{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run model module locally with unlabeled threshold tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Import os environment variables for global sequence shape hyperparameters\n",
    "os.environ[\"SEQ_LEN\"] = str(30)\n",
    "os.environ[\"NUM_FEAT\"] = str(5)\n",
    "\n",
    "# Import os environment variables for global feature hyperparameters\n",
    "os.environ[\"FEAT_NAMES\"] = (\",\").join([\"tag_{}\".format(i) for i in range(int(os.environ[\"NUM_FEAT\"]))])\n",
    "os.environ[\"FEAT_DEFAULTS\"] = (\",\").join([(\";\").join([\"0.0\"] * int(os.environ[\"SEQ_LEN\"]))] * int(os.environ[\"NUM_FEAT\"]))\n",
    "\n",
    "# Import os environment variables for global training hyperparameters \n",
    "os.environ[\"START_DELAY_SECS\"] = str(60)\n",
    "os.environ[\"THROTTLE_SECS\"] = str(120)\n",
    "\n",
    "# Import os environment variables for global threshold hyperparameters\n",
    "os.environ[\"LABELED_TUNE_THRESH\"] = \"False\"\n",
    "\n",
    "# Import global dense hyperparameters\n",
    "os.environ[\"ENC_DNN_HIDDEN_UNITS\"] = \"64,32,16\"\n",
    "os.environ[\"LATENT_VECTOR_SIZE\"] = str(8)\n",
    "os.environ[\"DEC_DNN_HIDDEN_UNITS\"] = \"16,32,64\"\n",
    "os.environ[\"TIME_LOSS_WEIGHT\"] = str(1.0)\n",
    "os.environ[\"FEAT_LOSS_WEIGHT\"] = str(1.0)\n",
    "\n",
    "# Import global lstm hyperparameters\n",
    "os.environ[\"REVERSE_LABELS_SEQUENCE\"] = \"True\"\n",
    "os.environ[\"ENC_LSTM_HIDDEN_UNITS\"] = \"64,32,16\"\n",
    "os.environ[\"DEC_LSTM_HIDDEN_UNITS\"] = \"16,32,64\"\n",
    "os.environ[\"LSTM_DROPOUT_OUTPUT_KEEP_PROBS\"] = \"0.9,0.95,1.0\"\n",
    "os.environ[\"DNN_HIDDEN_UNITS\"] = \"1024,256,64\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train reconstruction variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os environment variables for reconstruction training hyperparameters\n",
    "os.environ[\"TRAIN_FILE_PATTERN\"] = \"data/train_norm_seq.csv\"\n",
    "os.environ[\"EVAL_FILE_PATTERN\"] = \"data/val_norm_1_seq.csv\"\n",
    "os.environ[\"PREVIOUS_TRAIN_STEPS\"] = str(0)\n",
    "os.environ[\"RECONSTRUCTION_EPOCHS\"] = str(1.0)\n",
    "os.environ[\"TRAIN_EXAMPLES\"] = str(64000)\n",
    "os.environ[\"LEARNING_RATE\"] = str(0.1)\n",
    "os.environ[\"TRAINING_MODE\"] = \"reconstruction\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'latent_vector_size': 8, 'num_time_anom_thresh': 120, 'model_type': 'dense_autoencoder', 'feat_thresh_scl': 2.0, 'eval_batch_size': 32, 'max_time_anom_thresh': 2000.0, 'labeled_tune_thresh': False, 'time_thresh_scl': 2.0, 'training_mode': 'reconstruction', 'max_feat_anom_thresh': 2000.0, 'train_examples': 64000, 'num_feat_anom_thresh': 120, 'autotune_principal_components': False, 'enc_lstm_hidden_units': [64, 32, 16], 'train_batch_size': 32, 'train_file_pattern': 'data/train_norm_seq.csv', 'feat_anom_thresh': None, 'start_delay_secs': 60, 'num_feat': 5, 'dec_dnn_hidden_units': [16, 32, 64], 'min_feat_anom_thresh': 100.0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'seq_len': 30, 'dec_lstm_hidden_units': [16, 32, 64], 'time_anom_thresh': None, 'eval_examples': 1024, 'time_loss_weight': 1.0, 'learning_rate': 0.1, 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'k_principal_components_feat': None, 'reconstruction_epochs': 1.0, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'f_score_beta': 0.05, 'reverse_labels_sequence': True, 'feat_loss_weight': 1.0, 'min_time_anom_thresh': 100.0, 'throttle_secs': 120, 'enc_dnn_hidden_units': [64, 32, 16], 'dnn_hidden_units': [1024, 256, 64], 'previous_train_steps': 0, 'k_principal_components_time': None, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'eval_file_pattern': 'data/val_norm_1_seq.csv'}\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'latent_vector_size': 8, 'num_time_anom_thresh': 120, 'model_type': 'dense_autoencoder', 'feat_thresh_scl': 2.0, 'eval_batch_size': 32, 'max_time_anom_thresh': 2000.0, 'labeled_tune_thresh': False, 'time_thresh_scl': 2.0, 'training_mode': 'reconstruction', 'max_feat_anom_thresh': 2000.0, 'train_examples': 64000, 'num_feat_anom_thresh': 120, 'autotune_principal_components': False, 'enc_lstm_hidden_units': [64, 32, 16], 'train_batch_size': 32, 'train_file_pattern': 'data/train_norm_seq.csv', 'feat_anom_thresh': None, 'start_delay_secs': 60, 'num_feat': 5, 'dec_dnn_hidden_units': [16, 32, 64], 'min_feat_anom_thresh': 100.0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'seq_len': 30, 'dec_lstm_hidden_units': [16, 32, 64], 'time_anom_thresh': None, 'eval_examples': 1024, 'time_loss_weight': 1.0, 'learning_rate': 0.1, 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'k_principal_components_feat': None, 'reconstruction_epochs': 1.0, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'f_score_beta': 0.05, 'reverse_labels_sequence': True, 'feat_loss_weight': 1.0, 'min_time_anom_thresh': 100.0, 'throttle_secs': 120, 'enc_dnn_hidden_units': [64, 32, 16], 'dnn_hidden_units': [1024, 256, 64], 'previous_train_steps': 0, 'k_principal_components_time': None, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'eval_file_pattern': 'data/val_norm_1_seq.csv'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_evaluation_master': '', '_device_fn': None, '_service': None, '_task_type': 'worker', '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fb1a42be160>, '_save_checkpoints_steps': None, '_num_ps_replicas': 0, '_eval_distribute': None, '_is_chief': True, '_keep_checkpoint_max': 5, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_experimental_distribute': None, '_global_id_in_cluster': 0, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', '_log_step_count_steps': 100, '_train_distribute': None, '_task_id': 0, '_protocol': None, '_keep_checkpoint_every_n_hours': 10000, '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_dense.py:27: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:30:31.591827: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:30:31.598149: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:30:31.599606: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55ed84f2c280 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:30:31.599643: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.3404515, step = 1\n",
      "INFO:tensorflow:global_step/sec: 66.4802\n",
      "INFO:tensorflow:loss = 0.81656516, step = 101 (1.504 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.0628\n",
      "INFO:tensorflow:loss = 0.7967635, step = 201 (1.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.6413\n",
      "INFO:tensorflow:loss = 0.78774923, step = 301 (1.272 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.0811\n",
      "INFO:tensorflow:loss = 0.79237896, step = 401 (1.281 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.3686\n",
      "INFO:tensorflow:loss = 0.7996424, step = 501 (1.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.624\n",
      "INFO:tensorflow:loss = 0.7954267, step = 601 (1.288 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.5542\n",
      "INFO:tensorflow:loss = 0.791031, step = 701 (1.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.4907\n",
      "INFO:tensorflow:loss = 0.80373216, step = 801 (1.307 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.1406\n",
      "INFO:tensorflow:loss = 0.79523313, step = 901 (1.313 sec)\n",
      "INFO:tensorflow:global_step/sec: 74.3814\n",
      "INFO:tensorflow:loss = 0.80333966, step = 1001 (1.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 75.9354\n",
      "INFO:tensorflow:loss = 0.8038206, step = 1101 (1.317 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.4613\n",
      "INFO:tensorflow:loss = 0.7933268, step = 1201 (1.308 sec)\n",
      "INFO:tensorflow:global_step/sec: 74.8309\n",
      "INFO:tensorflow:loss = 0.7867508, step = 1301 (1.336 sec)\n",
      "INFO:tensorflow:global_step/sec: 75.4412\n",
      "INFO:tensorflow:loss = 0.798004, step = 1401 (1.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.6956\n",
      "INFO:tensorflow:loss = 0.7996515, step = 1501 (1.304 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.0607\n",
      "INFO:tensorflow:loss = 0.8009881, step = 1601 (1.315 sec)\n",
      "INFO:tensorflow:global_step/sec: 76.2282\n",
      "INFO:tensorflow:loss = 0.7892718, step = 1701 (1.312 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.2947\n",
      "INFO:tensorflow:loss = 0.803162, step = 1801 (1.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 77.3145\n",
      "INFO:tensorflow:loss = 0.814057, step = 1901 (1.294 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:31:01Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:31:03\n",
      "INFO:tensorflow:Saving dict for global step 2000: global_step = 2000, loss = 1.2981634, mae = 0.9461776, rmse = 1.1393697\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Loss for final step: 0.806937.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "rm -rf trained_model/dense_unlabeled\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/dense_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --reconstruction_epochs=$RECONSTRUCTION_EPOCHS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --learning_rate=$LEARNING_RATE \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"dense_autoencoder\" \\\n",
    "  --enc_dnn_hidden_units=$ENC_DNN_HIDDEN_UNITS \\\n",
    "  --latent_vector_size=$LATENT_VECTOR_SIZE \\\n",
    "  --dec_dnn_hidden_units=$DEC_DNN_HIDDEN_UNITS \\\n",
    "  --time_loss_weight=$TIME_LOSS_WEIGHT \\\n",
    "  --feat_loss_weight=$FEAT_LOSS_WEIGHT \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'latent_vector_size': 8, 'autotune_principal_components': False, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'train_examples': 64000, 'time_thresh_scl': 2.0, 'learning_rate': 0.1, 'start_delay_secs': 60, 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'reverse_labels_sequence': True, 'feat_thresh_scl': 2.0, 'max_time_anom_thresh': 2000.0, 'max_feat_anom_thresh': 2000.0, 'throttle_secs': 120, 'time_anom_thresh': None, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'eval_examples': 1024, 'num_time_anom_thresh': 120, 'training_mode': 'reconstruction', 'labeled_tune_thresh': False, 'train_file_pattern': 'data/train_norm_seq.csv', 'num_feat_anom_thresh': 120, 'time_loss_weight': 1.0, 'k_principal_components_feat': None, 'k_principal_components_time': None, 'dec_dnn_hidden_units': [64, 256, 1024], 'dec_lstm_hidden_units': [16, 32, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'min_time_anom_thresh': 100.0, 'previous_train_steps': 0, 'feat_anom_thresh': None, 'eps': 1e-12, 'min_feat_anom_thresh': 100.0, 'seq_len': 30, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'f_score_beta': 0.05, 'feat_loss_weight': 1.0, 'model_type': 'lstm_enc_dec_autoencoder', 'num_feat': 5, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'dnn_hidden_units': [1024, 256, 64], 'reconstruction_epochs': 1.0, 'train_batch_size': 32}\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'latent_vector_size': 8, 'autotune_principal_components': False, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'train_examples': 64000, 'time_thresh_scl': 2.0, 'learning_rate': 0.1, 'start_delay_secs': 60, 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'reverse_labels_sequence': True, 'feat_thresh_scl': 2.0, 'max_time_anom_thresh': 2000.0, 'max_feat_anom_thresh': 2000.0, 'throttle_secs': 120, 'time_anom_thresh': None, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'eval_examples': 1024, 'num_time_anom_thresh': 120, 'training_mode': 'reconstruction', 'labeled_tune_thresh': False, 'train_file_pattern': 'data/train_norm_seq.csv', 'num_feat_anom_thresh': 120, 'time_loss_weight': 1.0, 'k_principal_components_feat': None, 'k_principal_components_time': None, 'dec_dnn_hidden_units': [64, 256, 1024], 'dec_lstm_hidden_units': [16, 32, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'min_time_anom_thresh': 100.0, 'previous_train_steps': 0, 'feat_anom_thresh': None, 'eps': 1e-12, 'min_feat_anom_thresh': 100.0, 'seq_len': 30, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'f_score_beta': 0.05, 'feat_loss_weight': 1.0, 'model_type': 'lstm_enc_dec_autoencoder', 'num_feat': 5, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'dnn_hidden_units': [1024, 256, 64], 'reconstruction_epochs': 1.0, 'train_batch_size': 32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_global_id_in_cluster': 0, '_tf_random_seed': None, '_keep_checkpoint_max': 5, '_save_summary_steps': 100, '_protocol': None, '_is_chief': True, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_evaluation_master': '', '_num_worker_replicas': 1, '_num_ps_replicas': 0, '_eval_distribute': None, '_task_id': 0, '_save_checkpoints_steps': None, '_train_distribute': None, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_task_type': 'worker', '_master': '', '_experimental_distribute': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f694cf9b128>, '_device_fn': None, '_service': None, '_keep_checkpoint_every_n_hours': 10000}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:26: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:41: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:207: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:225: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:31:27.861307: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:31:27.868341: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:31:27.869722: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x563c78b1ead0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:31:27.869778: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.3610119, step = 1\n",
      "INFO:tensorflow:global_step/sec: 4.49297\n",
      "INFO:tensorflow:loss = 1.1821612, step = 101 (22.257 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7297\n",
      "INFO:tensorflow:loss = 1.1676013, step = 201 (7.283 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7295\n",
      "INFO:tensorflow:loss = 1.1580862, step = 301 (7.284 sec)\n",
      "INFO:tensorflow:global_step/sec: 12.1178\n",
      "INFO:tensorflow:loss = 1.1538372, step = 401 (8.252 sec)\n",
      "INFO:tensorflow:global_step/sec: 12.374\n",
      "INFO:tensorflow:loss = 1.1648186, step = 501 (8.082 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7699\n",
      "INFO:tensorflow:loss = 1.1954519, step = 601 (7.262 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.6346\n",
      "INFO:tensorflow:loss = 1.1733018, step = 701 (7.334 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7927\n",
      "INFO:tensorflow:loss = 1.1770564, step = 801 (7.250 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.796\n",
      "INFO:tensorflow:loss = 1.1693075, step = 901 (7.249 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7295\n",
      "INFO:tensorflow:loss = 1.1500492, step = 1001 (7.284 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.8181\n",
      "INFO:tensorflow:loss = 1.1448735, step = 1101 (7.237 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7046\n",
      "INFO:tensorflow:loss = 1.1400473, step = 1201 (7.297 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7715\n",
      "INFO:tensorflow:loss = 1.1266787, step = 1301 (7.261 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.6277\n",
      "INFO:tensorflow:loss = 1.1320038, step = 1401 (7.338 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.591\n",
      "INFO:tensorflow:loss = 1.12152, step = 1501 (7.358 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7111\n",
      "INFO:tensorflow:loss = 1.7890446, step = 1601 (7.293 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.7166\n",
      "INFO:tensorflow:loss = 1.5174981, step = 1701 (7.290 sec)\n",
      "INFO:tensorflow:global_step/sec: 12.6111\n",
      "INFO:tensorflow:loss = 1.3237267, step = 1801 (7.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.728\n",
      "INFO:tensorflow:loss = 1.2327219, step = 1901 (7.284 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:34:47Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:35:04\n",
      "INFO:tensorflow:Saving dict for global step 2000: global_step = 2000, loss = 1.1668096, mae = 0.9257265, rmse = 1.0801897\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Loss for final step: 1.1662966.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "rm -rf trained_model/lstm_unlabeled\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/lstm_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --reconstruction_epochs=$RECONSTRUCTION_EPOCHS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --learning_rate=$LEARNING_RATE \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"lstm_enc_dec_autoencoder\" \\\n",
    "  --reverse_labels_sequence=$REVERSE_LABELS_SEQUENCE \\\n",
    "  --enc_lstm_hidden_units=$ENC_LSTM_HIDDEN_UNITS \\\n",
    "  --dec_lstm_hidden_units=$DEC_LSTM_HIDDEN_UNITS \\\n",
    "  --lstm_dropout_output_keep_probs=$LSTM_DROPOUT_OUTPUT_KEEP_PROBS \\\n",
    "  --dnn_hidden_units=$DNN_HIDDEN_UNITS \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reconstruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'max_time_anom_thresh': 2000.0, 'eps': 1e-12, 'reverse_labels_sequence': True, 'dec_dnn_hidden_units': [64, 256, 1024], 'k_principal_components_time': None, 'dnn_hidden_units': [1024, 256, 64], 'max_feat_anom_thresh': 2000.0, 'labeled_tune_thresh': False, 'reconstruction_epochs': 1.0, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'time_loss_weight': 1.0, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'previous_train_steps': 0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'time_thresh_scl': 2.0, 'enc_dnn_hidden_units': [1024, 256, 64], 'start_delay_secs': 60, 'feat_thresh_scl': 2.0, 'dec_lstm_hidden_units': [16, 32, 64], 'min_time_anom_thresh': 100.0, 'eval_batch_size': 32, 'train_batch_size': 32, 'num_time_anom_thresh': 120, 'time_anom_thresh': None, 'model_type': 'pca', 'min_feat_anom_thresh': 100.0, 'eval_examples': 6400, 'train_file_pattern': 'data/train_norm_seq.csv', 'feat_anom_thresh': None, 'throttle_secs': 120, 'training_mode': 'reconstruction', 'autotune_principal_components': False, 'learning_rate': 0.1, 'num_feat_anom_thresh': 120, 'feat_loss_weight': 1.0, 'enc_lstm_hidden_units': [64, 32, 16], 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'seq_len': 30, 'k_principal_components_feat': None, 'num_feat': 5, 'latent_vector_size': 8, 'f_score_beta': 0.05, 'train_examples': 64000, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']]}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'max_time_anom_thresh': 2000.0, 'eps': 1e-12, 'reverse_labels_sequence': True, 'dec_dnn_hidden_units': [64, 256, 1024], 'k_principal_components_time': None, 'dnn_hidden_units': [1024, 256, 64], 'max_feat_anom_thresh': 2000.0, 'labeled_tune_thresh': False, 'reconstruction_epochs': 1.0, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'time_loss_weight': 1.0, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'previous_train_steps': 0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'time_thresh_scl': 2.0, 'enc_dnn_hidden_units': [1024, 256, 64], 'start_delay_secs': 60, 'feat_thresh_scl': 2.0, 'dec_lstm_hidden_units': [16, 32, 64], 'min_time_anom_thresh': 100.0, 'eval_batch_size': 32, 'train_batch_size': 32, 'num_time_anom_thresh': 120, 'time_anom_thresh': None, 'model_type': 'pca', 'min_feat_anom_thresh': 100.0, 'eval_examples': 6400, 'train_file_pattern': 'data/train_norm_seq.csv', 'feat_anom_thresh': None, 'throttle_secs': 120, 'training_mode': 'reconstruction', 'autotune_principal_components': False, 'learning_rate': 0.1, 'num_feat_anom_thresh': 120, 'feat_loss_weight': 1.0, 'enc_lstm_hidden_units': [64, 32, 16], 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'seq_len': 30, 'k_principal_components_feat': None, 'num_feat': 5, 'latent_vector_size': 8, 'f_score_beta': 0.05, 'train_examples': 64000, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_num_worker_replicas': 1, '_global_id_in_cluster': 0, '_save_checkpoints_steps': None, '_log_step_count_steps': 100, '_save_summary_steps': 100, '_eval_distribute': None, '_num_ps_replicas': 0, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fb3148b92b0>, '_tf_random_seed': None, '_service': None, '_task_type': 'worker', '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_protocol': None, '_train_distribute': None, '_device_fn': None, '_experimental_distribute': None, '_evaluation_master': '', '_master': '', '_save_checkpoints_secs': 600, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', '_is_chief': True, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      "}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:38:10.861251: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:38:10.867389: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:38:10.868231: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55c99fb70da0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:38:10.868265: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 1\n",
      "INFO:tensorflow:global_step/sec: 106.911\n",
      "INFO:tensorflow:loss = 0.0, step = 101 (0.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 124.023\n",
      "INFO:tensorflow:loss = 0.0, step = 201 (0.806 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.598\n",
      "INFO:tensorflow:loss = 0.0, step = 301 (0.816 sec)\n",
      "INFO:tensorflow:global_step/sec: 123.018\n",
      "INFO:tensorflow:loss = 0.0, step = 401 (0.813 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.859\n",
      "INFO:tensorflow:loss = 0.0, step = 501 (0.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 121.703\n",
      "INFO:tensorflow:loss = 0.0, step = 601 (0.822 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.38\n",
      "INFO:tensorflow:loss = 0.0, step = 701 (0.817 sec)\n",
      "INFO:tensorflow:global_step/sec: 115.72\n",
      "INFO:tensorflow:loss = 0.0, step = 801 (0.864 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.857\n",
      "INFO:tensorflow:loss = 0.0, step = 901 (0.871 sec)\n",
      "INFO:tensorflow:global_step/sec: 119.536\n",
      "INFO:tensorflow:loss = 0.0, step = 1001 (0.836 sec)\n",
      "INFO:tensorflow:global_step/sec: 122.829\n",
      "INFO:tensorflow:loss = 0.0, step = 1101 (0.814 sec)\n",
      "INFO:tensorflow:global_step/sec: 109.333\n",
      "INFO:tensorflow:loss = 0.0, step = 1201 (0.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 120.089\n",
      "INFO:tensorflow:loss = 0.0, step = 1301 (0.833 sec)\n",
      "INFO:tensorflow:global_step/sec: 127.81\n",
      "INFO:tensorflow:loss = 0.0, step = 1401 (0.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.94\n",
      "INFO:tensorflow:loss = 0.0, step = 1501 (0.788 sec)\n",
      "INFO:tensorflow:global_step/sec: 126.587\n",
      "INFO:tensorflow:loss = 0.0, step = 1601 (0.790 sec)\n",
      "INFO:tensorflow:global_step/sec: 128.661\n",
      "INFO:tensorflow:loss = 0.0, step = 1701 (0.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 144.097\n",
      "INFO:tensorflow:loss = 0.0, step = 1801 (0.694 sec)\n",
      "INFO:tensorflow:global_step/sec: 144.611\n",
      "INFO:tensorflow:loss = 0.0, step = 1901 (0.691 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:38:27Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:38:29\n",
      "INFO:tensorflow:Saving dict for global step 2000: global_step = 2000, loss = 0.6683831, mae = 0.6548333, rmse = 0.817547\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2000\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "rm -rf trained_model/pca_unlabeled\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/pca_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --reconstruction_epochs=1.0 \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --eval_examples=6400 \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"pca\" \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --autotune_principal_components=\"False\" \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Autotune principal components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'autotune_principal_components': False, 'min_time_anom_thresh': 100.0, 'k_principal_components_time': None, 'learning_rate': 0.1, 'time_loss_weight': 1.0, 'max_feat_anom_thresh': 2000.0, 'num_feat_anom_thresh': 120, 'labeled_tune_thresh': False, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'reverse_labels_sequence': True, 'eval_examples': 6400, 'dec_lstm_hidden_units': [16, 32, 64], 'k_principal_components_feat': None, 'max_time_anom_thresh': 2000.0, 'previous_train_steps': 2000, 'f_score_beta': 0.05, 'feat_thresh_scl': 2.0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'latent_vector_size': 8, 'min_feat_anom_thresh': 100.0, 'enc_dnn_hidden_units': [1024, 256, 64], 'eps': 1e-12, 'reconstruction_epochs': 1.0, 'train_examples': 6400, 'model_type': 'pca', 'feat_loss_weight': 1.0, 'time_thresh_scl': 2.0, 'num_time_anom_thresh': 120, 'time_anom_thresh': None, 'train_batch_size': 32, 'enc_lstm_hidden_units': [64, 32, 16], 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'start_delay_secs': 60, 'train_file_pattern': 'data/val_norm_1_seq.csv', 'training_mode': 'reconstruction', 'throttle_secs': 120, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'dec_dnn_hidden_units': [64, 256, 1024], 'num_feat': 5, 'eval_batch_size': 32, 'feat_anom_thresh': None, 'dnn_hidden_units': [1024, 256, 64], 'seq_len': 30}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'autotune_principal_components': False, 'min_time_anom_thresh': 100.0, 'k_principal_components_time': None, 'learning_rate': 0.1, 'time_loss_weight': 1.0, 'max_feat_anom_thresh': 2000.0, 'num_feat_anom_thresh': 120, 'labeled_tune_thresh': False, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'reverse_labels_sequence': True, 'eval_examples': 6400, 'dec_lstm_hidden_units': [16, 32, 64], 'k_principal_components_feat': None, 'max_time_anom_thresh': 2000.0, 'previous_train_steps': 2000, 'f_score_beta': 0.05, 'feat_thresh_scl': 2.0, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'latent_vector_size': 8, 'min_feat_anom_thresh': 100.0, 'enc_dnn_hidden_units': [1024, 256, 64], 'eps': 1e-12, 'reconstruction_epochs': 1.0, 'train_examples': 6400, 'model_type': 'pca', 'feat_loss_weight': 1.0, 'time_thresh_scl': 2.0, 'num_time_anom_thresh': 120, 'time_anom_thresh': None, 'train_batch_size': 32, 'enc_lstm_hidden_units': [64, 32, 16], 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'start_delay_secs': 60, 'train_file_pattern': 'data/val_norm_1_seq.csv', 'training_mode': 'reconstruction', 'throttle_secs': 120, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'dec_dnn_hidden_units': [64, 256, 1024], 'num_feat': 5, 'eval_batch_size': 32, 'feat_anom_thresh': None, 'dnn_hidden_units': [1024, 256, 64], 'seq_len': 30}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_train_distribute': None, '_is_chief': True, '_tf_random_seed': None, '_eval_distribute': None, '_save_checkpoints_steps': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f9384206198>, '_protocol': None, '_evaluation_master': '', '_keep_checkpoint_every_n_hours': 10000, '_global_id_in_cluster': 0, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_master': '', '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', '_save_summary_steps': 100, '_device_fn': None, '_num_worker_replicas': 1, '_experimental_distribute': None, '_task_id': 0, '_num_ps_replicas': 0, '_task_type': 'worker', '_log_step_count_steps': 100, '_service': None, '_save_checkpoints_secs': 600, '_keep_checkpoint_max': 5}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:38:33.607815: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:38:33.613911: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:38:33.615087: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55b7c42fc8b0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:38:33.615138: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2000\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2001\n",
      "INFO:tensorflow:global_step/sec: 111.973\n",
      "INFO:tensorflow:loss = 0.0, step = 2101 (0.894 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:38:36Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:38:38\n",
      "INFO:tensorflow:Saving dict for global step 2200: global_step = 2200, loss = 0.6683827, mae = 0.65482765, rmse = 0.8175468\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2200: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/pca_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=2000 \\\n",
    "  --reconstruction_epochs=1.0 \\\n",
    "  --train_examples=6400 \\\n",
    "  --eval_examples=6400 \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"pca\" \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --autotune_principal_components=\"True\" \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train error distribution statistics variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os environment variables for error dist training hyperparameters\n",
    "os.environ[\"TRAIN_FILE_PATTERN\"] = \"data/val_norm_1_seq.csv\"\n",
    "os.environ[\"EVAL_FILE_PATTERN\"] = \"data/val_norm_1_seq.csv\"\n",
    "os.environ[\"PREVIOUS_TRAIN_STEPS\"] = str(2000)\n",
    "os.environ[\"TRAIN_EXAMPLES\"] = str(6400)\n",
    "os.environ[\"TRAINING_MODE\"] = \"calculate_error_distribution_statistics\"\n",
    "os.environ[\"EPS\"] = \"1e-12\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_batch_size': 32, 'time_loss_weight': 1.0, 'labeled_tune_thresh': False, 'feat_loss_weight': 1.0, 'seq_len': 30, 'eps': 1e-12, 'max_feat_anom_thresh': 2000.0, 'enc_lstm_hidden_units': [64, 32, 16], 'train_examples': 6400, 'enc_dnn_hidden_units': [64, 32, 16], 'reconstruction_epochs': 1.0, 'eval_examples': 1024, 'training_mode': 'calculate_error_distribution_statistics', 'throttle_secs': 120, 'dec_dnn_hidden_units': [16, 32, 64], 'num_time_anom_thresh': 120, 'start_delay_secs': 60, 'dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'time_thresh_scl': 2.0, 'min_time_anom_thresh': 100.0, 'f_score_beta': 0.05, 'num_feat': 5, 'previous_train_steps': 2000, 'dec_lstm_hidden_units': [16, 32, 64], 'num_feat_anom_thresh': 120, 'feat_thresh_scl': 2.0, 'reverse_labels_sequence': True, 'train_file_pattern': 'data/val_norm_1_seq.csv', 'k_principal_components_feat': None, 'k_principal_components_time': None, 'model_type': 'dense_autoencoder', 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'time_anom_thresh': None, 'latent_vector_size': 8, 'eval_batch_size': 32, 'max_time_anom_thresh': 2000.0, 'learning_rate': 0.1, 'min_feat_anom_thresh': 100.0, 'autotune_principal_components': False, 'feat_anom_thresh': None}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_batch_size': 32, 'time_loss_weight': 1.0, 'labeled_tune_thresh': False, 'feat_loss_weight': 1.0, 'seq_len': 30, 'eps': 1e-12, 'max_feat_anom_thresh': 2000.0, 'enc_lstm_hidden_units': [64, 32, 16], 'train_examples': 6400, 'enc_dnn_hidden_units': [64, 32, 16], 'reconstruction_epochs': 1.0, 'eval_examples': 1024, 'training_mode': 'calculate_error_distribution_statistics', 'throttle_secs': 120, 'dec_dnn_hidden_units': [16, 32, 64], 'num_time_anom_thresh': 120, 'start_delay_secs': 60, 'dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'time_thresh_scl': 2.0, 'min_time_anom_thresh': 100.0, 'f_score_beta': 0.05, 'num_feat': 5, 'previous_train_steps': 2000, 'dec_lstm_hidden_units': [16, 32, 64], 'num_feat_anom_thresh': 120, 'feat_thresh_scl': 2.0, 'reverse_labels_sequence': True, 'train_file_pattern': 'data/val_norm_1_seq.csv', 'k_principal_components_feat': None, 'k_principal_components_time': None, 'model_type': 'dense_autoencoder', 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'time_anom_thresh': None, 'latent_vector_size': 8, 'eval_batch_size': 32, 'max_time_anom_thresh': 2000.0, 'learning_rate': 0.1, 'min_feat_anom_thresh': 100.0, 'autotune_principal_components': False, 'feat_anom_thresh': None}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_keep_checkpoint_max': 5, '_log_step_count_steps': 100, '_evaluation_master': '', '_num_worker_replicas': 1, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fb6d6950390>, '_task_id': 0, '_task_type': 'worker', '_experimental_distribute': None, '_num_ps_replicas': 0, '_train_distribute': None, '_is_chief': True, '_device_fn': None, '_protocol': None, '_keep_checkpoint_every_n_hours': 10000, '_eval_distribute': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_service': None, '_save_checkpoints_secs': 600, '_master': '', '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', '_save_checkpoints_steps': None, '_global_id_in_cluster': 0, '_save_summary_steps': 100, '_tf_random_seed': None}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_dense.py:27: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:39:33.393817: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:39:33.401436: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:39:33.403417: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55b9095874f0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:39:33.403481: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2000\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2001\n",
      "INFO:tensorflow:global_step/sec: 99.3533\n",
      "INFO:tensorflow:loss = 0.0, step = 2101 (1.007 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:39:36Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:39:38\n",
      "INFO:tensorflow:Saving dict for global step 2200: global_step = 2200, loss = 1.2981634\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2200: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/dense_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"dense_autoencoder\" \\\n",
    "  --enc_dnn_hidden_units=$ENC_DNN_HIDDEN_UNITS \\\n",
    "  --latent_vector_size=$LATENT_VECTOR_SIZE \\\n",
    "  --dec_dnn_hidden_units=$DEC_DNN_HIDDEN_UNITS \\\n",
    "  --time_loss_weight=$TIME_LOSS_WEIGHT \\\n",
    "  --feat_loss_weight=$FEAT_LOSS_WEIGHT \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --eps=$EPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'train_file_pattern': 'data/val_norm_1_seq.csv', 'reverse_labels_sequence': True, 'throttle_secs': 120, 'previous_train_steps': 2000, 'f_score_beta': 0.05, 'model_type': 'lstm_enc_dec_autoencoder', 'k_principal_components_feat': None, 'time_loss_weight': 1.0, 'time_anom_thresh': None, 'eval_examples': 1024, 'dnn_hidden_units': [1024, 256, 64], 'reconstruction_epochs': 1.0, 'autotune_principal_components': False, 'latent_vector_size': 8, 'training_mode': 'calculate_error_distribution_statistics', 'seq_len': 30, 'max_feat_anom_thresh': 2000.0, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'dec_lstm_hidden_units': [16, 32, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'min_time_anom_thresh': 100.0, 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'learning_rate': 0.1, 'max_time_anom_thresh': 2000.0, 'dec_dnn_hidden_units': [64, 256, 1024], 'enc_dnn_hidden_units': [1024, 256, 64], 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'start_delay_secs': 60, 'num_feat': 5, 'feat_anom_thresh': None, 'train_examples': 6400, 'train_batch_size': 32, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'feat_loss_weight': 1.0, 'num_feat_anom_thresh': 120, 'time_thresh_scl': 2.0, 'feat_thresh_scl': 2.0, 'num_time_anom_thresh': 120, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'min_feat_anom_thresh': 100.0, 'labeled_tune_thresh': False, 'eval_batch_size': 32, 'k_principal_components_time': None}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'train_file_pattern': 'data/val_norm_1_seq.csv', 'reverse_labels_sequence': True, 'throttle_secs': 120, 'previous_train_steps': 2000, 'f_score_beta': 0.05, 'model_type': 'lstm_enc_dec_autoencoder', 'k_principal_components_feat': None, 'time_loss_weight': 1.0, 'time_anom_thresh': None, 'eval_examples': 1024, 'dnn_hidden_units': [1024, 256, 64], 'reconstruction_epochs': 1.0, 'autotune_principal_components': False, 'latent_vector_size': 8, 'training_mode': 'calculate_error_distribution_statistics', 'seq_len': 30, 'max_feat_anom_thresh': 2000.0, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'dec_lstm_hidden_units': [16, 32, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'min_time_anom_thresh': 100.0, 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'learning_rate': 0.1, 'max_time_anom_thresh': 2000.0, 'dec_dnn_hidden_units': [64, 256, 1024], 'enc_dnn_hidden_units': [1024, 256, 64], 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'start_delay_secs': 60, 'num_feat': 5, 'feat_anom_thresh': None, 'train_examples': 6400, 'train_batch_size': 32, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'feat_loss_weight': 1.0, 'num_feat_anom_thresh': 120, 'time_thresh_scl': 2.0, 'feat_thresh_scl': 2.0, 'num_time_anom_thresh': 120, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'min_feat_anom_thresh': 100.0, 'labeled_tune_thresh': False, 'eval_batch_size': 32, 'k_principal_components_time': None}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_tf_random_seed': None, '_global_id_in_cluster': 0, '_log_step_count_steps': 100, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_experimental_distribute': None, '_num_ps_replicas': 0, '_keep_checkpoint_every_n_hours': 10000, '_evaluation_master': '', '_eval_distribute': None, '_service': None, '_save_summary_steps': 100, '_task_type': 'worker', '_train_distribute': None, '_protocol': None, '_num_worker_replicas': 1, '_master': '', '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', '_keep_checkpoint_max': 5, '_task_id': 0, '_is_chief': True, '_device_fn': None, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f311cde33c8>}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:26: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:41: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:207: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:225: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:39:48.545144: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:39:48.551379: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:39:48.552664: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55bd46c0a7c0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:39:48.552693: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2000\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2000 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2001\n",
      "INFO:tensorflow:global_step/sec: 9.51453\n",
      "INFO:tensorflow:loss = 0.0, step = 2101 (10.511 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:40:17Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:40:36\n",
      "INFO:tensorflow:Saving dict for global step 2200: global_step = 2200, loss = 1.1668096\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2200: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2200\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/lstm_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"lstm_enc_dec_autoencoder\" \\\n",
    "  --reverse_labels_sequence=$REVERSE_LABELS_SEQUENCE \\\n",
    "  --enc_lstm_hidden_units=$ENC_LSTM_HIDDEN_UNITS \\\n",
    "  --dec_lstm_hidden_units=$DEC_LSTM_HIDDEN_UNITS \\\n",
    "  --lstm_dropout_output_keep_probs=$LSTM_DROPOUT_OUTPUT_KEEP_PROBS \\\n",
    "  --dnn_hidden_units=$DNN_HIDDEN_UNITS \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --eps=$EPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'time_loss_weight': 1.0, 'feat_anom_thresh': None, 'min_feat_anom_thresh': 100.0, 'max_feat_anom_thresh': 2000.0, 'learning_rate': 0.1, 'autotune_principal_components': False, 'enc_lstm_hidden_units': [64, 32, 16], 'seq_len': 30, 'time_thresh_scl': 2.0, 'time_anom_thresh': None, 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'num_time_anom_thresh': 120, 'max_time_anom_thresh': 2000.0, 'min_time_anom_thresh': 100.0, 'model_type': 'pca', 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'dec_lstm_hidden_units': [16, 32, 64], 'train_batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'f_score_beta': 0.05, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_file_pattern': 'data/val_norm_1_seq.csv', 'throttle_secs': 120, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'num_feat_anom_thresh': 120, 'feat_thresh_scl': 2.0, 'reconstruction_epochs': 1.0, 'num_feat': 5, 'k_principal_components_time': None, 'latent_vector_size': 8, 'start_delay_secs': 60, 'k_principal_components_feat': None, 'dec_dnn_hidden_units': [64, 256, 1024], 'reverse_labels_sequence': True, 'feat_loss_weight': 1.0, 'train_examples': 6400, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'previous_train_steps': 2200, 'labeled_tune_thresh': False, 'eval_examples': 1024, 'training_mode': 'calculate_error_distribution_statistics'}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'time_loss_weight': 1.0, 'feat_anom_thresh': None, 'min_feat_anom_thresh': 100.0, 'max_feat_anom_thresh': 2000.0, 'learning_rate': 0.1, 'autotune_principal_components': False, 'enc_lstm_hidden_units': [64, 32, 16], 'seq_len': 30, 'time_thresh_scl': 2.0, 'time_anom_thresh': None, 'eps': 1e-12, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'num_time_anom_thresh': 120, 'max_time_anom_thresh': 2000.0, 'min_time_anom_thresh': 100.0, 'model_type': 'pca', 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'dec_lstm_hidden_units': [16, 32, 64], 'train_batch_size': 32, 'dnn_hidden_units': [1024, 256, 64], 'f_score_beta': 0.05, 'eval_file_pattern': 'data/val_norm_1_seq.csv', 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_file_pattern': 'data/val_norm_1_seq.csv', 'throttle_secs': 120, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'num_feat_anom_thresh': 120, 'feat_thresh_scl': 2.0, 'reconstruction_epochs': 1.0, 'num_feat': 5, 'k_principal_components_time': None, 'latent_vector_size': 8, 'start_delay_secs': 60, 'k_principal_components_feat': None, 'dec_dnn_hidden_units': [64, 256, 1024], 'reverse_labels_sequence': True, 'feat_loss_weight': 1.0, 'train_examples': 6400, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'previous_train_steps': 2200, 'labeled_tune_thresh': False, 'eval_examples': 1024, 'training_mode': 'calculate_error_distribution_statistics'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_log_step_count_steps': 100, '_num_worker_replicas': 1, '_experimental_distribute': None, '_service': None, '_eval_distribute': None, '_global_id_in_cluster': 0, '_train_distribute': None, '_is_chief': True, '_num_ps_replicas': 0, '_save_checkpoints_secs': 600, '_evaluation_master': '', '_task_id': 0, '_task_type': 'worker', '_device_fn': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_every_n_hours': 10000, '_keep_checkpoint_max': 5, '_save_checkpoints_steps': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5ad3de6320>, '_tf_random_seed': None, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', '_save_summary_steps': 100, '_master': '', '_protocol': None}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:40:42.001785: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:40:42.008729: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:40:42.009757: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55fac13340c0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:40:42.009792: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2200\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2201\n",
      "INFO:tensorflow:global_step/sec: 157.375\n",
      "INFO:tensorflow:loss = 0.0, step = 2301 (0.636 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2400 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/losses/losses_impl.py:667: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:40:44Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2400\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:40:45\n",
      "INFO:tensorflow:Saving dict for global step 2400: global_step = 2400, loss = 0.6683827\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2400: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2400\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/pca_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=2200 \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"pca\" \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --eps=$EPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tune anomaly thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os environment variables for tune threshold training hyperparameters\n",
    "os.environ[\"PREVIOUS_TRAIN_STEPS\"] = str(2200)\n",
    "os.environ[\"TRAIN_EXAMPLES\"] = str(12800)\n",
    "os.environ[\"TRAINING_MODE\"] = \"tune_anomaly_thresholds\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unlabeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os environment variables for unlabeled tune threshold training hyperparameters\n",
    "os.environ[\"TRAIN_FILE_PATTERN\"] = \"data/unlabeled_val_mixed_seq.csv\"\n",
    "os.environ[\"EVAL_FILE_PATTERN\"] = \"data/unlabeled_val_mixed_seq.csv\"\n",
    "os.environ[\"TIME_THRESH_SCL\"] = str(2.0)\n",
    "os.environ[\"FEAT_THRESH_SCL\"] = str(2.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'throttle_secs': 120, 'latent_vector_size': 8, 'min_time_anom_thresh': 100.0, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'k_principal_components_feat': None, 'autotune_principal_components': False, 'seq_len': 30, 'dec_lstm_hidden_units': [16, 32, 64], 'train_examples': 12800, 'num_feat_anom_thresh': 120, 'num_time_anom_thresh': 120, 'dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'labeled_tune_thresh': False, 'max_time_anom_thresh': 2000.0, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'eps': 1e-12, 'eval_examples': 1024, 'model_type': 'dense_autoencoder', 'learning_rate': 0.1, 'time_loss_weight': 1.0, 'reverse_labels_sequence': True, 'reconstruction_epochs': 1.0, 'min_feat_anom_thresh': 100.0, 'start_delay_secs': 60, 'enc_dnn_hidden_units': [64, 32, 16], 'feat_thresh_scl': 2.0, 'f_score_beta': 0.05, 'num_feat': 5, 'time_anom_thresh': None, 'training_mode': 'tune_anomaly_thresholds', 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'k_principal_components_time': None, 'dec_dnn_hidden_units': [16, 32, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'time_thresh_scl': 2.0, 'feat_loss_weight': 1.0, 'previous_train_steps': 2200, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'enc_lstm_hidden_units': [64, 32, 16], 'max_feat_anom_thresh': 2000.0, 'train_batch_size': 32, 'feat_anom_thresh': None}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'throttle_secs': 120, 'latent_vector_size': 8, 'min_time_anom_thresh': 100.0, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'k_principal_components_feat': None, 'autotune_principal_components': False, 'seq_len': 30, 'dec_lstm_hidden_units': [16, 32, 64], 'train_examples': 12800, 'num_feat_anom_thresh': 120, 'num_time_anom_thresh': 120, 'dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'labeled_tune_thresh': False, 'max_time_anom_thresh': 2000.0, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'eps': 1e-12, 'eval_examples': 1024, 'model_type': 'dense_autoencoder', 'learning_rate': 0.1, 'time_loss_weight': 1.0, 'reverse_labels_sequence': True, 'reconstruction_epochs': 1.0, 'min_feat_anom_thresh': 100.0, 'start_delay_secs': 60, 'enc_dnn_hidden_units': [64, 32, 16], 'feat_thresh_scl': 2.0, 'f_score_beta': 0.05, 'num_feat': 5, 'time_anom_thresh': None, 'training_mode': 'tune_anomaly_thresholds', 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'k_principal_components_time': None, 'dec_dnn_hidden_units': [16, 32, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'time_thresh_scl': 2.0, 'feat_loss_weight': 1.0, 'previous_train_steps': 2200, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'enc_lstm_hidden_units': [64, 32, 16], 'max_feat_anom_thresh': 2000.0, 'train_batch_size': 32, 'feat_anom_thresh': None}\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_1': <tf.Tensor 'StringToNumber:0' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'StringToNumber_1:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'StringToNumber_2:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'StringToNumber_3:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'StringToNumber_4:0' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "infer\n",
      "anomaly_detection: params = \n",
      "{'throttle_secs': 120, 'latent_vector_size': 8, 'min_time_anom_thresh': 100.0, 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'k_principal_components_feat': None, 'autotune_principal_components': False, 'seq_len': 30, 'dec_lstm_hidden_units': [16, 32, 64], 'train_examples': 12800, 'num_feat_anom_thresh': 120, 'num_time_anom_thresh': 120, 'dnn_hidden_units': [1024, 256, 64], 'eval_batch_size': 32, 'labeled_tune_thresh': False, 'max_time_anom_thresh': 2000.0, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'eps': 1e-12, 'eval_examples': 1024, 'model_type': 'dense_autoencoder', 'learning_rate': 0.1, 'time_loss_weight': 1.0, 'reverse_labels_sequence': True, 'reconstruction_epochs': 1.0, 'min_feat_anom_thresh': 100.0, 'start_delay_secs': 60, 'enc_dnn_hidden_units': [64, 32, 16], 'feat_thresh_scl': 2.0, 'f_score_beta': 0.05, 'num_feat': 5, 'time_anom_thresh': None, 'training_mode': 'tune_anomaly_thresholds', 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'k_principal_components_time': None, 'dec_dnn_hidden_units': [16, 32, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'time_thresh_scl': 2.0, 'feat_loss_weight': 1.0, 'previous_train_steps': 2200, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'enc_lstm_hidden_units': [64, 32, 16], 'max_feat_anom_thresh': 2000.0, 'train_batch_size': 32, 'feat_anom_thresh': None}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_evaluation_master': '', '_save_checkpoints_steps': None, '_eval_distribute': None, '_tf_random_seed': None, '_keep_checkpoint_max': 5, '_master': '', '_keep_checkpoint_every_n_hours': 10000, '_protocol': None, '_task_type': 'worker', '_task_id': 0, '_save_checkpoints_secs': 600, '_num_ps_replicas': 0, '_is_chief': True, '_service': None, '_num_worker_replicas': 1, '_device_fn': None, '_train_distribute': None, '_experimental_distribute': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_log_step_count_steps': 100, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fefe7ac32e8>, '_save_summary_steps': 100, '_global_id_in_cluster': 0}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_dense.py:27: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:40:49.740206: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:40:49.745779: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:40:49.746646: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55f487e59b00 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:40:49.746672: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2200\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2201\n",
      "INFO:tensorflow:global_step/sec: 141.625\n",
      "INFO:tensorflow:loss = 0.0, step = 2301 (0.707 sec)\n",
      "INFO:tensorflow:global_step/sec: 170.301\n",
      "INFO:tensorflow:loss = 0.0, step = 2401 (0.587 sec)\n",
      "INFO:tensorflow:global_step/sec: 166.723\n",
      "INFO:tensorflow:loss = 0.0, step = 2501 (0.600 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2600 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/metrics_impl.py:363: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:40:53Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:40:56\n",
      "INFO:tensorflow:Saving dict for global step 2600: feat_anom_tp = 0.031640626, global_step = 2600, loss = 0.0, time_anom_tp = 0.4996875\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2600: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/dense_unlabeled/export/exporter/temp-b'1562964056'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/dense_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"dense_autoencoder\" \\\n",
    "  --enc_dnn_hidden_units=$ENC_DNN_HIDDEN_UNITS \\\n",
    "  --latent_vector_size=$LATENT_VECTOR_SIZE \\\n",
    "  --dec_dnn_hidden_units=$DEC_DNN_HIDDEN_UNITS \\\n",
    "  --time_loss_weight=$TIME_LOSS_WEIGHT \\\n",
    "  --feat_loss_weight=$FEAT_LOSS_WEIGHT \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --time_thresh_scl=$TIME_THRESH_SCL \\\n",
    "  --feat_thresh_scl=$FEAT_THRESH_SCL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'k_principal_components_feat': None, 'num_feat': 5, 'eps': 1e-12, 'feat_thresh_scl': 2.0, 'seq_len': 30, 'time_loss_weight': 1.0, 'latent_vector_size': 8, 'labeled_tune_thresh': False, 'num_time_anom_thresh': 120, 'start_delay_secs': 60, 'throttle_secs': 120, 'feat_anom_thresh': None, 'min_feat_anom_thresh': 100.0, 'max_feat_anom_thresh': 2000.0, 'eval_batch_size': 32, 'train_batch_size': 32, 'eval_examples': 1024, 'model_type': 'lstm_enc_dec_autoencoder', 'learning_rate': 0.1, 'dec_dnn_hidden_units': [64, 256, 1024], 'feat_loss_weight': 1.0, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'dnn_hidden_units': [1024, 256, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'time_anom_thresh': None, 'previous_train_steps': 2200, 'min_time_anom_thresh': 100.0, 'dec_lstm_hidden_units': [16, 32, 64], 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'k_principal_components_time': None, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'reconstruction_epochs': 1.0, 'train_examples': 12800, 'reverse_labels_sequence': True, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'training_mode': 'tune_anomaly_thresholds', 'autotune_principal_components': False, 'num_feat_anom_thresh': 120, 'time_thresh_scl': 2.0, 'f_score_beta': 0.05, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'max_time_anom_thresh': 2000.0}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'k_principal_components_feat': None, 'num_feat': 5, 'eps': 1e-12, 'feat_thresh_scl': 2.0, 'seq_len': 30, 'time_loss_weight': 1.0, 'latent_vector_size': 8, 'labeled_tune_thresh': False, 'num_time_anom_thresh': 120, 'start_delay_secs': 60, 'throttle_secs': 120, 'feat_anom_thresh': None, 'min_feat_anom_thresh': 100.0, 'max_feat_anom_thresh': 2000.0, 'eval_batch_size': 32, 'train_batch_size': 32, 'eval_examples': 1024, 'model_type': 'lstm_enc_dec_autoencoder', 'learning_rate': 0.1, 'dec_dnn_hidden_units': [64, 256, 1024], 'feat_loss_weight': 1.0, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'dnn_hidden_units': [1024, 256, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'time_anom_thresh': None, 'previous_train_steps': 2200, 'min_time_anom_thresh': 100.0, 'dec_lstm_hidden_units': [16, 32, 64], 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'k_principal_components_time': None, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'reconstruction_epochs': 1.0, 'train_examples': 12800, 'reverse_labels_sequence': True, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'training_mode': 'tune_anomaly_thresholds', 'autotune_principal_components': False, 'num_feat_anom_thresh': 120, 'time_thresh_scl': 2.0, 'f_score_beta': 0.05, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'max_time_anom_thresh': 2000.0}\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_3': <tf.Tensor 'StringToNumber:0' shape=(?, 30) dtype=float64>, 'tag_0': <tf.Tensor 'StringToNumber_1:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'StringToNumber_2:0' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'StringToNumber_3:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'StringToNumber_4:0' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "infer\n",
      "anomaly_detection: params = \n",
      "{'k_principal_components_feat': None, 'num_feat': 5, 'eps': 1e-12, 'feat_thresh_scl': 2.0, 'seq_len': 30, 'time_loss_weight': 1.0, 'latent_vector_size': 8, 'labeled_tune_thresh': False, 'num_time_anom_thresh': 120, 'start_delay_secs': 60, 'throttle_secs': 120, 'feat_anom_thresh': None, 'min_feat_anom_thresh': 100.0, 'max_feat_anom_thresh': 2000.0, 'eval_batch_size': 32, 'train_batch_size': 32, 'eval_examples': 1024, 'model_type': 'lstm_enc_dec_autoencoder', 'learning_rate': 0.1, 'dec_dnn_hidden_units': [64, 256, 1024], 'feat_loss_weight': 1.0, 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'dnn_hidden_units': [1024, 256, 64], 'enc_lstm_hidden_units': [64, 32, 16], 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'time_anom_thresh': None, 'previous_train_steps': 2200, 'min_time_anom_thresh': 100.0, 'dec_lstm_hidden_units': [16, 32, 64], 'lstm_dropout_output_keep_probs': [0.9, 0.95, 1.0], 'k_principal_components_time': None, 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', 'reconstruction_epochs': 1.0, 'train_examples': 12800, 'reverse_labels_sequence': True, 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'training_mode': 'tune_anomaly_thresholds', 'autotune_principal_components': False, 'num_feat_anom_thresh': 120, 'time_thresh_scl': 2.0, 'f_score_beta': 0.05, 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'max_time_anom_thresh': 2000.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f749fc522b0>, '_task_id': 0, '_master': '', '_log_step_count_steps': 100, '_experimental_distribute': None, '_tf_random_seed': None, '_eval_distribute': None, '_evaluation_master': '', '_service': None, '_global_id_in_cluster': 0, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/', '_save_checkpoints_secs': 600, '_train_distribute': None, '_keep_checkpoint_every_n_hours': 10000, '_save_summary_steps': 100, '_is_chief': True, '_task_type': 'worker', '_keep_checkpoint_max': 5, '_device_fn': None, '_num_ps_replicas': 0, '_save_checkpoints_steps': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_protocol': None, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:26: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:41: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:207: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/autoencoder_lstm.py:225: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:41:06.694066: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:41:06.717815: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:41:06.719114: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55faf1ea86c0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:41:06.719147: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2200\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2200 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2201\n",
      "INFO:tensorflow:global_step/sec: 9.93121\n",
      "INFO:tensorflow:loss = 0.0, step = 2301 (10.070 sec)\n",
      "INFO:tensorflow:global_step/sec: 13.094\n",
      "INFO:tensorflow:loss = 0.0, step = 2401 (7.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 12.8895\n",
      "INFO:tensorflow:loss = 0.0, step = 2501 (7.758 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2600 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/metrics_impl.py:363: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:41:51Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:42:23\n",
      "INFO:tensorflow:Saving dict for global step 2600: feat_anom_tp = 0.004140625, global_step = 2600, loss = 0.0, time_anom_tp = 0.5\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2600: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/model.ckpt-2600\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/lstm_unlabeled/export/exporter/temp-b'1562964144'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/lstm_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=$PREVIOUS_TRAIN_STEPS \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"lstm_enc_dec_autoencoder\" \\\n",
    "  --reverse_labels_sequence=$REVERSE_LABELS_SEQUENCE \\\n",
    "  --enc_lstm_hidden_units=$ENC_LSTM_HIDDEN_UNITS \\\n",
    "  --dec_lstm_hidden_units=$DEC_LSTM_HIDDEN_UNITS \\\n",
    "  --lstm_dropout_output_keep_probs=$LSTM_DROPOUT_OUTPUT_KEEP_PROBS \\\n",
    "  --dnn_hidden_units=$DNN_HIDDEN_UNITS \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --time_thresh_scl=$TIME_THRESH_SCL \\\n",
    "  --feat_thresh_scl=$FEAT_THRESH_SCL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "train\n",
      "anomaly_detection: params = \n",
      "{'feat_thresh_scl': 2.0, 'dec_dnn_hidden_units': [64, 256, 1024], 'num_feat_anom_thresh': 120, 'seq_len': 30, 'latent_vector_size': 8, 'throttle_secs': 120, 'time_loss_weight': 1.0, 'dec_lstm_hidden_units': [16, 32, 64], 'reconstruction_epochs': 1.0, 'previous_train_steps': 2400, 'eval_examples': 1024, 'model_type': 'pca', 'time_anom_thresh': None, 'start_delay_secs': 60, 'autotune_principal_components': False, 'min_time_anom_thresh': 100.0, 'max_time_anom_thresh': 2000.0, 'k_principal_components_time': None, 'train_examples': 12800, 'eval_batch_size': 32, 'learning_rate': 0.1, 'max_feat_anom_thresh': 2000.0, 'min_feat_anom_thresh': 100.0, 'dnn_hidden_units': [1024, 256, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'feat_loss_weight': 1.0, 'eps': 1e-12, 'enc_lstm_hidden_units': [64, 32, 16], 'labeled_tune_thresh': False, 'k_principal_components_feat': None, 'training_mode': 'tune_anomaly_thresholds', 'f_score_beta': 0.05, 'num_feat': 5, 'reverse_labels_sequence': True, 'feat_anom_thresh': None, 'num_time_anom_thresh': 120, 'train_batch_size': 32, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'time_thresh_scl': 2.0}\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'IteratorGetNext:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'IteratorGetNext:4' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'IteratorGetNext:2' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'IteratorGetNext:3' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'IteratorGetNext:1' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "eval\n",
      "anomaly_detection: params = \n",
      "{'feat_thresh_scl': 2.0, 'dec_dnn_hidden_units': [64, 256, 1024], 'num_feat_anom_thresh': 120, 'seq_len': 30, 'latent_vector_size': 8, 'throttle_secs': 120, 'time_loss_weight': 1.0, 'dec_lstm_hidden_units': [16, 32, 64], 'reconstruction_epochs': 1.0, 'previous_train_steps': 2400, 'eval_examples': 1024, 'model_type': 'pca', 'time_anom_thresh': None, 'start_delay_secs': 60, 'autotune_principal_components': False, 'min_time_anom_thresh': 100.0, 'max_time_anom_thresh': 2000.0, 'k_principal_components_time': None, 'train_examples': 12800, 'eval_batch_size': 32, 'learning_rate': 0.1, 'max_feat_anom_thresh': 2000.0, 'min_feat_anom_thresh': 100.0, 'dnn_hidden_units': [1024, 256, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'feat_loss_weight': 1.0, 'eps': 1e-12, 'enc_lstm_hidden_units': [64, 32, 16], 'labeled_tune_thresh': False, 'k_principal_components_feat': None, 'training_mode': 'tune_anomaly_thresholds', 'f_score_beta': 0.05, 'num_feat': 5, 'reverse_labels_sequence': True, 'feat_anom_thresh': None, 'num_time_anom_thresh': 120, 'train_batch_size': 32, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'time_thresh_scl': 2.0}\n",
      "\n",
      "anomaly_detection: features = \n",
      "{'tag_0': <tf.Tensor 'StringToNumber_1:0' shape=(?, 30) dtype=float64>, 'tag_4': <tf.Tensor 'StringToNumber_2:0' shape=(?, 30) dtype=float64>, 'tag_2': <tf.Tensor 'StringToNumber_3:0' shape=(?, 30) dtype=float64>, 'tag_3': <tf.Tensor 'StringToNumber:0' shape=(?, 30) dtype=float64>, 'tag_1': <tf.Tensor 'StringToNumber_4:0' shape=(?, 30) dtype=float64>}\n",
      "anomaly_detection: labels = \n",
      "None\n",
      "anomaly_detection: mode = \n",
      "infer\n",
      "anomaly_detection: params = \n",
      "{'feat_thresh_scl': 2.0, 'dec_dnn_hidden_units': [64, 256, 1024], 'num_feat_anom_thresh': 120, 'seq_len': 30, 'latent_vector_size': 8, 'throttle_secs': 120, 'time_loss_weight': 1.0, 'dec_lstm_hidden_units': [16, 32, 64], 'reconstruction_epochs': 1.0, 'previous_train_steps': 2400, 'eval_examples': 1024, 'model_type': 'pca', 'time_anom_thresh': None, 'start_delay_secs': 60, 'autotune_principal_components': False, 'min_time_anom_thresh': 100.0, 'max_time_anom_thresh': 2000.0, 'k_principal_components_time': None, 'train_examples': 12800, 'eval_batch_size': 32, 'learning_rate': 0.1, 'max_feat_anom_thresh': 2000.0, 'min_feat_anom_thresh': 100.0, 'dnn_hidden_units': [1024, 256, 64], 'feat_names': ['tag_0', 'tag_1', 'tag_2', 'tag_3', 'tag_4'], 'feat_loss_weight': 1.0, 'eps': 1e-12, 'enc_lstm_hidden_units': [64, 32, 16], 'labeled_tune_thresh': False, 'k_principal_components_feat': None, 'training_mode': 'tune_anomaly_thresholds', 'f_score_beta': 0.05, 'num_feat': 5, 'reverse_labels_sequence': True, 'feat_anom_thresh': None, 'num_time_anom_thresh': 120, 'train_batch_size': 32, 'lstm_dropout_output_keep_probs': [1.0, 1.0, 1.0], 'train_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'feat_defaults': [['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0'], ['0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0;0.0']], 'enc_dnn_hidden_units': [1024, 256, 64], 'eval_file_pattern': 'data/unlabeled_val_mixed_seq.csv', 'output_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', 'time_thresh_scl': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_keep_checkpoint_every_n_hours': 10000, '_save_checkpoints_steps': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fec0be16278>, '_protocol': None, '_global_id_in_cluster': 0, '_tf_random_seed': None, '_train_distribute': None, '_task_id': 0, '_model_dir': '/home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/', '_num_worker_replicas': 1, '_task_type': 'worker', '_service': None, '_eval_distribute': None, '_device_fn': None, '_is_chief': True, '_experimental_distribute': None, '_save_summary_steps': 100, '_num_ps_replicas': 0, '_master': '', '_evaluation_master': '', '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_keep_checkpoint_max': 5, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      "}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2019-07-12 20:42:34.785091: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2019-07-12 20:42:34.791707: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:42:34.793069: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x56221739e990 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:42:34.793119: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2400\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/training/saver.py:1070: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file utilities to get mtimes.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2400 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.0, step = 2401\n",
      "INFO:tensorflow:global_step/sec: 100.344\n",
      "INFO:tensorflow:loss = 0.0, step = 2501 (0.997 sec)\n",
      "INFO:tensorflow:global_step/sec: 114.837\n",
      "INFO:tensorflow:loss = 0.0, step = 2601 (0.871 sec)\n",
      "INFO:tensorflow:global_step/sec: 113.541\n",
      "INFO:tensorflow:loss = 0.0, step = 2701 (0.881 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2800 into /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/metrics_impl.py:363: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-07-12T20:42:39Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2800\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-07-12-20:42:42\n",
      "INFO:tensorflow:Saving dict for global step 2800: feat_anom_tp = 0.041171875, global_step = 2800, loss = 0.0, time_anom_tp = 0.4999219\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2800: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2800\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:205: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/model.ckpt-2800\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /home/jupyter/artificial_intelligence/machine_learning/anomaly_detection/tf_anomaly_detection_model_selection/trained_model/pca_unlabeled/export/exporter/temp-b'1562964162'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/anomaly_detection_module\n",
    "python3 -m trainer.task \\\n",
    "  --train_file_pattern=$TRAIN_FILE_PATTERN \\\n",
    "  --eval_file_pattern=$EVAL_FILE_PATTERN \\\n",
    "  --output_dir=$PWD/trained_model/pca_unlabeled \\\n",
    "  --job-dir=./tmp \\\n",
    "  --seq_len=$SEQ_LEN \\\n",
    "  --num_feat=$NUM_FEAT \\\n",
    "  --feat_names=$FEAT_NAMES \\\n",
    "  --feat_defaults=$FEAT_DEFAULTS \\\n",
    "  --train_batch_size=32 \\\n",
    "  --eval_batch_size=32 \\\n",
    "  --previous_train_steps=2400 \\\n",
    "  --train_examples=$TRAIN_EXAMPLES \\\n",
    "  --start_delay_secs=$START_DELAY_SECS \\\n",
    "  --throttle_secs=$THROTTLE_SECS \\\n",
    "  --model_type=\"pca\" \\\n",
    "  --training_mode=$TRAINING_MODE \\\n",
    "  --labeled_tune_thresh=$LABELED_TUNE_THRESH \\\n",
    "  --time_thresh_scl=$TIME_THRESH_SCL \\\n",
    "  --feat_thresh_scl=$FEAT_THRESH_SCL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNLABELED_CSV_COLUMNS = [\"tag_{0}\".format(tag) for tag in range(0, 5)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unlabeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unlabeled_test_mixed_sequences_array.shape = (12800, 5)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "unlabeled_test_mixed_sequences_array = np.loadtxt(\n",
    "    fname=\"data/unlabeled_test_mixed_seq.csv\", dtype=str, delimiter=\",\")\n",
    "print(\"unlabeled_test_mixed_sequences_array.shape = {}\".format(\n",
    "    unlabeled_test_mixed_sequences_array.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labels = ['0.10912429;1.21779374;1.52916082;-0.177806;-0.54337799;0.44055714;1.27923318;1.18207574;-0.84995686;0.08906497;0.65940998;1.61871863;0.81842511;-0.96566587;-0.60382206;1.69003285;1.66798859;0.31696774;-0.408903;-0.25624363;1.39733346;1.38432466;-0.19920215;-0.79272972;0.38229627;1.13707338;1.69781059;0.19309917;-0.97771496;0.44386982'\n",
      " '0.14320316;1.29770315;1.07947085;-0.1310581;-0.3940327;1.23110166;1.25224647;0.83662982;-0.94877962;-0.00280088;0.60816257;1.16871775;0.67355164;-0.33326375;-0.16736942;0.78481252;1.17160361;0.34967432;-0.97550557;0.52889076;1.63690058;1.67796765;-0.1170558;-0.95527216;0.50201129;1.89823695;1.12541774;0.27771255;-0.11889077;0.54388248'\n",
      " '0.92380376;1.1275517;0.68680086;-0.65989216;-0.65966357;1.13504582;1.71752066;1.16837965;-0.71586137;-0.13441075;1.14169827;1.99655389;0.69922314;-0.6801096;0.29903466;1.68053185;1.2336307;0.32646373;-1.03204426;-0.33344275;1.11825307;26.42256512;-7.4768817;6.55099231;-1.56550118;23.40616989;-12.96872267;0.5533737;3.96783856;3.01739029'\n",
      " '0.57473342;1.14567792;0.76228677;-0.52907114;-0.68788196;0.32984607;2.00630745;0.64677117;-0.69549809;-0.65562508;0.65211815;1.63234544;0.27237479;-0.88844836;-0.55278754;0.9332511;1.30141274;0.36581009;-0.43250124;0.46858911;0.97705268;1.9008542;-0.14739552;-0.32990692;0.53638701;1.03819819;0.92842024;0.00824748;-0.73914075;0.39245204'\n",
      " '0.21639525;1.29097127;0.724104;-0.61433765;-0.00680535;0.92093701;1.87805548;0.80229314;-0.14227465;-0.10884144;0.92188529;2.05525766;0.65663376;-0.2425855;-0.24612095;0.9582669;1.15462945;0.26062798;-1.01275174;-0.06147401;1.67518222;1.49463349;-0.3449311;-0.70891745;0.2578082;1.50897842;1.26550938;-0.07863006;-0.69447811;1.00209485'\n",
      " '0.07684221;1.43927545;0.7009711;-0.67108844;-0.27681638;0.77290783;1.64031508;0.7206991;-0.570293;-0.05605199;0.57439664;1.31988135;0.31988395;-1.06125243;-0.5133299;1.17022203;1.3979644;0.15867121;-0.3304164;0.10197166;1.76667587;-22.26722639;-4.35478848;-11.86683088;-0.23064614;-10.9364861;10.47263779;3.58262026;-2.89953151;-9.82481352'\n",
      " '0.71816782;1.93279224;1.1308937;-0.76595683;-0.06663918;1.24122717;1.91919796;0.44800218;-0.12060529;-0.06574122;0.98898873;1.38518081;0.17085414;-0.23837417;-0.0814766;1.02935793;1.43180106;0.65021482;-0.41917392;-0.34275552;0.93253301;1.16802359;-0.39598329;-0.65709446;-0.12743336;1.40998721;1.29330737;-0.25137872;-0.2269882;0.09304095'\n",
      " '0.85844291;1.20242051;1.22971499;-0.68325236;-0.68442464;0.499797;1.27085339;1.03038335;-0.48198528;-0.79360583;1.11030659;1.34817054;0.13874612;-0.67617502;0.3399162;1.30852939;1.72673467;0.73408344;-0.36569274;0.39278922;1.89621852;21.87142178;1.25477499;1.88732616;8.75886641;-17.98581112;-16.83071132;0.22739148;13.99363541;12.16146561'\n",
      " '0.31762637;1.3804855;0.74013553;0.08044917;-0.42999489;1.02955808;2.10122827;1.19921475;-0.45532116;-0.05713612;0.79972361;1.44591912;0.10363462;-0.42901693;-0.50389135;1.38943358;1.09777739;0.24972303;-1.02435044;-0.35454585;1.73304027;1.38327166;0.33709665;-1.07568816;0.35512227;1.87887447;0.73001908;0.0153715;-0.08519375;0.34442849'\n",
      " '0.73287365;1.36180986;1.43065531;-0.10370082;-0.4443194;0.34385391;1.68280177;0.58334118;-0.19710264;-0.20602948;1.46444107;1.52896532;0.72828249;-0.2093811;-0.14278271;1.30636694;1.33975716;-0.06555973;-0.6946629;0.42337979;1.78894264;-26.2773364;-2.52555829;-17.74821318;2.91317954;24.31799318;-10.33218681;6.86935028;3.68243741;18.87684323']\n"
     ]
    }
   ],
   "source": [
    "number_of_prediction_instances = 10\n",
    "print(\"labels = {}\".format(\n",
    "  unlabeled_test_mixed_sequences_array[0:number_of_prediction_instances, -1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local prediction from local model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"tag_0\": \"0.08630851;2.37045866;1.20743555;-1.14710842;-0.48609804;1.50719389;2.19601683;0.10701992;-1.24018965;-0.19057113;1.8352755;1.752288;-0.68841143;-1.25716387;0.38385502;2.6536265;0.79187743;-1.04243811;-0.52800537;1.76135306;2.42560225;0.47596221;-0.93880634;0.34620397;1.77331762;1.74868177;-0.03435227;-0.88146609;0.34844452;2.28759207\", \"tag_1\": \"0.01710703;1.43010645;-0.40862645;0.54163604;2.06510263;-0.00407659;-0.12560559;1.91115068;0.08341453;-0.99214653;1.59915954;0.90426161;-1.14516381;0.92210796;0.52287498;-0.74294574;0.70845168;1.03833225;-0.4001544;0.95978758;1.08263277;-0.31584913;0.00862613;1.30877146;-0.22048323;-0.24728125;1.4809888;-0.20183649;-0.16185678;1.24482918\", \"tag_2\": \"0.94729535;2.04044194;1.47843099;0.16625841;-0.75320741;-0.41766578;1.258229;1.52247393;0.72566737;-0.61741024;-0.55178713;0.43867216;1.09453834;1.59920523;0.34086948;-0.8572519;-0.9924474;0.89896643;1.69688929;1.0605381;-0.45109315;-0.96045705;-0.59540668;1.24977148;2.10291277;0.88623454;-0.87614424;-0.54596172;0.32117828;1.82773091\", \"tag_3\": \"0.39874107;2.00116413;0.31937228;-0.16407445;1.40468418;0.85744544;-0.96783301;0.09800176;1.56246727;0.24066002;-0.20638952;1.49479056;0.8453935;-0.65794037;0.81768388;1.53697035;-0.60762906;-0.41178175;1.97426229;1.27885467;-0.99065918;0.75421874;2.28180555;-0.0776754;-0.52754049;2.00615088;0.92636092;-1.06403935;0.65159031;2.25599744\", \"tag_4\": \"0.10912429;1.21779374;1.52916082;-0.177806;-0.54337799;0.44055714;1.27923318;1.18207574;-0.84995686;0.08906497;0.65940998;1.61871863;0.81842511;-0.96566587;-0.60382206;1.69003285;1.66798859;0.31696774;-0.408903;-0.25624363;1.39733346;1.38432466;-0.19920215;-0.79272972;0.38229627;1.13707338;1.69781059;0.19309917;-0.97771496;0.44386982\"}\n",
      "{\"tag_0\": \"0.33326098;1.85213809;0.9367061;-0.70338119;-1.04182581;1.76439127;2.36971889;0.02464679;-1.33960764;0.03777979;2.43879841;2.07333333;-0.63452483;-0.80411878;0.68186144;2.09670113;0.71299439;-1.27104652;-0.93329058;1.77084673;1.75081246;-0.12435358;-1.22616821;-0.14038669;1.9363168;1.57056515;-0.93900226;-1.15834962;1.15378587;1.90757666\", \"tag_1\": \"0.69319743;1.99742394;-0.34534823;-0.18763758;1.98480288;0.04597313;-0.60822359;2.07220695;0.58939806;-0.50059845;1.30255595;0.29817767;-0.66609921;1.75415711;1.18993307;-0.82581176;0.65695489;1.58478922;-1.12820328;0.37838228;1.41823725;-0.80229066;-0.10926555;1.55461754;-0.40455603;-0.35484316;1.67598159;0.20869392;-0.4338172;1.67318093\", \"tag_2\": \"0.01836919;2.10079674;1.45575831;0.4292372;-1.18797205;-0.80021461;0.8894862;1.75786206;1.46853713;-0.02519642;-0.9383983;0.12175777;1.4568485;1.43912591;0.9491389;-1.20018373;-0.35509279;0.95545257;2.21985516;1.78304166;-0.13381787;-0.64774564;-0.23528506;1.2975945;2.208059;0.63649609;-0.68651097;-0.76814667;0.51465272;1.5418206\", \"tag_3\": \"0.34392828;2.22022254;-0.62486174;-0.64590716;1.6803571;0.75811967;-1.26129641;0.88613567;1.55815841;-0.2753162;-0.69940586;1.89796118;1.53808388;-1.39588061;0.46195093;2.09925021;-0.23745644;-0.79142007;1.28753278;0.76307192;-0.45385005;0.92661575;1.42590031;-0.69073748;-0.85690587;1.28799933;0.49766533;-0.98356903;0.32941458;1.87212755\", \"tag_4\": \"0.14320316;1.29770315;1.07947085;-0.1310581;-0.3940327;1.23110166;1.25224647;0.83662982;-0.94877962;-0.00280088;0.60816257;1.16871775;0.67355164;-0.33326375;-0.16736942;0.78481252;1.17160361;0.34967432;-0.97550557;0.52889076;1.63690058;1.67796765;-0.1170558;-0.95527216;0.50201129;1.89823695;1.12541774;0.27771255;-0.11889077;0.54388248\"}\n",
      "{\"tag_0\": \"0.73445717;2.72353904;0.88211537;-1.32664507;-0.5210361;1.29496037;1.90929892;0.4232938;-0.80236621;-0.70083891;1.52373953;1.83750916;-0.3666452;-1.5214794;0.98290438;2.48461036;0.96254537;-1.21225544;-0.93638252;1.11062498;2.34966482;-0.35560748;23.20449342;3.8036013;-25.57941473;-15.92919286;-15.76944922;-14.0063463;5.42956756;19.74947804\", \"tag_1\": \"0.19593637;1.76879507;-0.83715153;0.59350303;1.63039714;0.18211453;0.17978823;2.12240027;0.19751445;-0.36542212;1.57107445;0.9932112;-0.53727715;0.91906189;0.53024939;-0.77351041;0.6343829;1.34788263;-0.53486089;0.99252583;1.70572377;5.34229905;-1.93771172;24.33740572;3.09886769;5.4089372;22.0932672;-5.37011838;11.2843526;15.25303853\", \"tag_2\": \"0.75526572;1.66106826;2.08408904;0.52117267;-0.46353711;-0.66013605;1.15982625;1.64075277;1.28945757;-0.19133592;-0.65200834;0.4360381;1.88170736;2.01106892;1.0584947;-0.78742361;-0.76440265;0.74252922;1.94999506;1.95534266;-0.42487493;11.34676376;-8.75499856;13.73333075;30.43334572;16.02409734;-12.42682254;-17.83573912;3.73843064;17.66327856\", \"tag_3\": \"0.10518585;1.97269593;0.05511016;-0.36149535;1.3647047;1.4665965;-0.98610863;0.72499472;2.19375337;-0.48821122;-0.56973544;1.2726906;0.59708671;-1.14598304;0.17213327;1.78199422;0.17972115;-0.81727977;2.15765594;1.30375861;-0.76766416;-4.5207422;-38.77923254;-0.25832894;12.26882665;19.72439461;16.23403127;7.62880641;4.28571629;-17.2711947\", \"tag_4\": \"0.92380376;1.1275517;0.68680086;-0.65989216;-0.65966357;1.13504582;1.71752066;1.16837965;-0.71586137;-0.13441075;1.14169827;1.99655389;0.69922314;-0.6801096;0.29903466;1.68053185;1.2336307;0.32646373;-1.03204426;-0.33344275;1.11825307;26.42256512;-7.4768817;6.55099231;-1.56550118;23.40616989;-12.96872267;0.5533737;3.96783856;3.01739029\"}\n",
      "{\"tag_0\": \"0.82922283;1.93741843;1.49492381;-0.84450981;-0.99175216;1.20327578;2.50783232;0.74062725;-0.97837255;-0.1600501;2.07475931;2.11854126;-0.8516401;-1.63834829;0.38282412;2.66203786;1.09576432;-0.93283521;-0.70167395;1.6087528;2.27045246;0.22035135;-1.34514546;0.25842238;1.84306938;1.61957121;-0.64942144;-1.65355856;1.02095443;2.29193653\", \"tag_1\": \"0.03141424;1.87318302;-0.75120973;0.37567858;1.79430594;0.24884296;-0.3549795;2.03170349;0.3524101;-0.12704528;1.41145559;0.78698945;-0.79337177;1.22171682;0.75325722;-0.63854466;0.86144481;1.80888452;-0.57026325;0.41910896;1.94140258;-0.54408244;0.39212857;2.05022252;0.14915739;0.12620143;2.06180843;0.01903747;-0.65691172;1.38681956\", \"tag_2\": \"0.8769017;1.92795571;1.34776236;0.2965219;-0.83390422;-0.22458234;1.54375522;1.40369793;1.46911887;-0.32129543;-1.04940981;-0.05921177;1.49446773;1.87888029;1.08123109;-1.18643694;-1.03703208;1.10419775;1.64290886;1.55647337;0.31549323;-0.70443465;-0.64984686;1.40289051;1.41239503;0.66957773;-0.40790381;-1.20435453;0.51918987;1.77076249\", \"tag_3\": \"0.74079055;1.98100402;-0.42935497;-0.60234479;1.35627275;1.13012176;-0.97505105;0.31526484;1.80576032;0.25040035;-0.87448263;1.46366832;0.96609291;-1.08672155;0.6199302;1.48714591;-0.48262486;-0.14518067;1.74607215;0.8593231;-0.68618833;0.95924103;1.53021779;-0.05102364;-0.30358979;1.73928984;1.28427223;-1.37103933;0.64565625;1.8640818\", \"tag_4\": \"0.57473342;1.14567792;0.76228677;-0.52907114;-0.68788196;0.32984607;2.00630745;0.64677117;-0.69549809;-0.65562508;0.65211815;1.63234544;0.27237479;-0.88844836;-0.55278754;0.9332511;1.30141274;0.36581009;-0.43250124;0.46858911;0.97705268;1.9008542;-0.14739552;-0.32990692;0.53638701;1.03819819;0.92842024;0.00824748;-0.73914075;0.39245204\"}\n",
      "{\"tag_0\": \"0.00395872;2.60235251;1.5949603;-0.71084193;-1.33850556;1.31221383;2.1709366;0.02467309;-1.06122985;-0.59989862;2.18405289;1.52798934;-0.00668257;-1.51419615;0.38615404;1.88242164;1.67594433;-1.16729733;-0.51237519;1.17367911;2.48567458;0.40346433;-1.73140114;0.13266756;2.49172216;2.0067257;-0.86826965;-1.45925123;0.56729308;2.35972169\", \"tag_1\": \"0.17322761;1.26088587;-0.44106297;0.08205769;1.87909407;-0.04154736;0.20789049;1.44724909;0.46116592;-0.5750887;1.11220246;1.0131389;-0.66804451;1.75556884;0.62338149;-0.39225775;0.6553524;0.87031166;-1.02781636;1.13821784;1.50961073;-0.41324001;0.09332299;1.43290083;-0.60220055;0.17632098;1.73060641;0.11357086;-0.29357654;1.72154401\", \"tag_2\": \"0.26589048;1.40844799;1.67682136;-0.03902525;-0.81559788;-0.74917911;0.80962982;1.58878076;1.48562317;-0.19949458;-0.81230424;-0.30197958;1.93347315;2.21363113;0.28217837;-1.0483017;-0.93107819;0.36755138;1.44745005;1.21499395;0.08662917;-1.09115891;-0.4442448;0.93969392;1.7698406;0.4989453;-0.61616411;-1.05891473;0.59688692;1.47360871\", \"tag_3\": \"0.19148424;2.06885428;-0.29010287;-0.1467994;1.65619133;0.59808712;-1.28491792;0.11253167;1.63342552;-0.40367216;-0.14003887;1.78758198;1.40968721;-0.98866213;1.05421333;1.74642712;-0.4191681;-0.62040339;1.7590535;1.44448295;-1.06671911;0.16977676;1.42139252;0.22087214;-0.19544175;2.20376187;1.27066737;-0.82935885;1.16824215;2.23016063\", \"tag_4\": \"0.21639525;1.29097127;0.724104;-0.61433765;-0.00680535;0.92093701;1.87805548;0.80229314;-0.14227465;-0.10884144;0.92188529;2.05525766;0.65663376;-0.2425855;-0.24612095;0.9582669;1.15462945;0.26062798;-1.01275174;-0.06147401;1.67518222;1.49463349;-0.3449311;-0.70891745;0.2578082;1.50897842;1.26550938;-0.07863006;-0.69447811;1.00209485\"}\n",
      "{\"tag_0\": \"0.04198157;2.40464693;0.83422681;-1.28655387;-0.4486482;0.90924833;2.00919569;0.35177161;-1.19195108;0.07913664;1.69191395;2.27105765;-0.82989558;-1.32748633;0.96047968;2.07146584;1.30113328;-1.13774912;-0.92650173;0.96896607;2.2479503;-2.99136024;-10.37769835;-0.09257541;28.04971053;16.90729127;-13.01427036;-19.99435482;4.82956485;-30.00292139\", \"tag_1\": \"0.5445968;1.93855543;-0.79309737;0.19743954;1.62495129;0.2170903;-0.72407506;1.68839448;0.1620954;-0.05562843;1.51895767;0.31144909;-0.80047731;1.48580891;0.69011396;-0.30338762;1.2909128;1.59033414;-0.58210036;0.97229196;1.27090013;-1.88694647;8.8323189;-33.82369936;9.37830517;-0.65613327;24.39548948;6.06445066;4.18754522;20.00937916\", \"tag_2\": \"0.74713393;1.64342955;1.87344957;0.64392896;-0.60540843;-0.26079101;0.60141338;1.95012321;0.80444699;-0.45761676;-0.91015551;0.14063373;1.79404715;2.14000609;0.09987011;-0.9679956;-0.89017087;0.87491112;1.96371061;1.85660303;0.4351123;20.41215846;-7.96100284;-11.15526335;-25.47050852;-16.73292722;1.65518979;-18.29737582;15.43341168;-14.57786136\", \"tag_3\": \"0.41999368;2.35518789;-0.25778947;-0.80550182;1.26143821;1.1320193;-0.54196054;0.73724908;1.76571016;-0.08080697;-0.646462;1.96670358;1.00162974;-0.45770954;0.105134;2.32310232;-0.19204034;-0.37593684;2.18621105;1.2904411;-0.89710548;-3.32734437;37.6425341;-3.56347617;7.42552103;21.61633275;15.20761755;-16.44548885;17.51024662;-18.23183368\", \"tag_4\": \"0.07684221;1.43927545;0.7009711;-0.67108844;-0.27681638;0.77290783;1.64031508;0.7206991;-0.570293;-0.05605199;0.57439664;1.31988135;0.31988395;-1.06125243;-0.5133299;1.17022203;1.3979644;0.15867121;-0.3304164;0.10197166;1.76667587;-22.26722639;-4.35478848;-11.86683088;-0.23064614;-10.9364861;10.47263779;3.58262026;-2.89953151;-9.82481352\"}\n",
      "{\"tag_0\": \"0.26843148;2.26033987;1.46649872;-1.18559026;-1.01489562;1.34284786;1.81892725;0.62455093;-1.09421145;-0.22209055;2.135944;1.63843107;-0.54887201;-1.19015976;1.02346538;2.36435974;1.15563113;-1.33679828;-1.19139794;1.39472245;2.28870725;0.47593012;-1.64843334;0.17878788;1.69174971;2.24072223;-0.35276955;-0.91466379;0.42032311;2.22452248\", \"tag_1\": \"0.0129413;1.25257419;-0.63354585;0.14772502;1.98985502;0.20805635;-0.15564056;2.02559049;0.63256353;-0.60072758;1.64876388;0.71485497;-0.61395614;1.57488881;1.39285713;-1.0018011;0.76538498;0.85770778;-0.29570332;0.59699765;1.29575314;-0.42318787;0.59362552;1.27317238;-0.1596227;-0.1570993;1.28972811;-0.1626878;0.04428588;1.45567135\", \"tag_2\": \"0.84167842;1.41036038;1.8918916;0.45974852;-0.7343536;0.00460531;1.55599581;2.16319655;1.52948075;-0.27410036;-1.19466723;0.26222559;1.44656176;1.54485897;0.93423596;-0.67878524;-0.47525442;0.56632129;1.41027762;0.97055232;0.48786689;-1.20517916;-0.02615132;0.81730794;1.45372776;0.77646932;-0.16390073;-0.3935256;0.80687563;1.25003549\", \"tag_3\": \"0.51668829;1.48923535;-0.4130304;-0.8569307;1.67864363;0.96157018;-0.44961473;0.95740493;1.51357334;-0.52634693;-0.48855733;1.82967798;0.64401769;-0.88344649;0.43085387;1.57572669;-0.28971486;-0.68328878;1.90457909;1.08835561;-0.46744631;0.624297;1.44398575;-0.3663162;-0.74575323;2.149788;1.37136336;-0.48076731;0.2749356;1.61791733\", \"tag_4\": \"0.71816782;1.93279224;1.1308937;-0.76595683;-0.06663918;1.24122717;1.91919796;0.44800218;-0.12060529;-0.06574122;0.98898873;1.38518081;0.17085414;-0.23837417;-0.0814766;1.02935793;1.43180106;0.65021482;-0.41917392;-0.34275552;0.93253301;1.16802359;-0.39598329;-0.65709446;-0.12743336;1.40998721;1.29330737;-0.25137872;-0.2269882;0.09304095\"}\n",
      "{\"tag_0\": \"0.89180226;2.31624275;1.42900981;-1.18392361;-1.25722046;0.95304286;2.76033728;0.69800503;-1.45664554;-0.22755546;2.39072737;2.10810524;-0.8478035;-1.46539578;0.51741038;2.47064072;1.05393861;-1.17411598;-1.19996648;0.97610328;2.56799413;-3.543338;-15.18411178;-3.72931002;26.25904703;35.6764447;-8.55514354;22.60172941;-18.2564907;30.39933748\", \"tag_1\": \"0.84318528;1.86597104;-0.21976836;0.09125206;1.8176748;0.08197284;-0.16188409;1.84861597;0.60879015;-0.4849663;1.82883446;1.10875741;-0.51875078;1.56315002;1.01035818;-0.50342552;0.93220307;1.80177097;-0.89111713;0.77078174;1.21356016;-17.63090091;1.72919318;-20.90292918;6.32788852;-1.52999812;21.00848186;-0.96300052;3.85340831;-30.4646747\", \"tag_2\": \"0.8540975;1.90076489;1.55675081;0.05960444;-1.12935042;-0.61829833;0.74441137;2.06917428;1.56569194;-0.24322329;-0.3942758;0.21063404;1.45739488;1.6837368;0.14842067;-0.51201288;-0.90351649;1.08800035;2.26679095;1.80498611;0.45884521;-4.36797955;-10.5543302;-12.8227163;-23.6433477;-23.95030157;-7.97897494;11.24913645;9.45412606;26.01480599\", \"tag_3\": \"0.83445521;1.56656846;-0.1175664;-0.24832165;1.76970234;1.25819298;-0.73492293;0.56792263;1.74924589;-0.5440741;-0.76928938;1.53635771;0.90918041;-0.50749604;0.72920564;1.90364445;-0.49282908;-0.28137251;1.76892769;1.44923507;-1.12993775;-20.63993453;37.50970576;-9.2109456;-10.80014883;14.86353444;-8.10343951;19.41943538;-4.3992938;-43.59556627\", \"tag_4\": \"0.85844291;1.20242051;1.22971499;-0.68325236;-0.68442464;0.499797;1.27085339;1.03038335;-0.48198528;-0.79360583;1.11030659;1.34817054;0.13874612;-0.67617502;0.3399162;1.30852939;1.72673467;0.73408344;-0.36569274;0.39278922;1.89621852;21.87142178;1.25477499;1.88732616;8.75886641;-17.98581112;-16.83071132;0.22739148;13.99363541;12.16146561\"}\n",
      "{\"tag_0\": \"0.15337954;2.01708229;0.93291083;-0.79588847;-1.33391811;1.63407951;1.78791304;0.29064493;-0.9693213;0.10825243;2.04250757;2.29730801;-0.299841;-0.81875728;0.59409408;1.88104282;1.35365175;-0.90467911;-0.54092312;1.46684523;2.45530682;0.24984035;-1.17295784;0.23211731;1.64433242;1.951103;-0.37166115;-0.76779075;0.53672622;2.50630976\", \"tag_1\": \"0.36430147;1.42140005;-0.15052786;0.36975754;1.37599528;-0.56632877;0.16482797;2.14542972;0.51511198;-0.85607982;1.36853124;0.53237315;-0.33560107;1.76377169;1.02266286;-0.51965387;0.93491768;0.82101721;-0.4511027;1.034781;1.52995214;-0.07021636;0.38759908;1.93082931;-0.0797287;0.11664255;1.73262589;-0.163324;-0.52521025;2.15850457\", \"tag_2\": \"0.15909093;2.18877796;1.74357021;0.15424528;-1.14390818;-0.89461909;1.26173858;1.52397546;0.76956137;-0.70301387;-1.1953103;-0.29794095;1.82918778;1.45365066;0.66730892;-1.01679012;-0.3992378;1.13579279;1.44196465;1.51803771;0.21883592;-1.10407414;0.24003135;1.21265229;1.61535209;1.15981526;-0.57104637;-1.06882048;0.07114742;1.23608828\", \"tag_3\": \"0.57958877;2.18701699;-0.01000531;-0.91034762;2.00516971;1.15890536;-0.49372638;0.52346325;2.2807258;-0.17254453;-0.8618773;1.62107646;0.69358897;-0.90358565;0.59307121;1.90669763;-0.02578955;-0.53776964;1.53817706;0.88018757;-0.66633308;0.77500638;1.45668506;-0.04197625;-0.05076989;1.91579399;0.50196923;-0.42649724;0.72888643;2.0544927\", \"tag_4\": \"0.31762637;1.3804855;0.74013553;0.08044917;-0.42999489;1.02955808;2.10122827;1.19921475;-0.45532116;-0.05713612;0.79972361;1.44591912;0.10363462;-0.42901693;-0.50389135;1.38943358;1.09777739;0.24972303;-1.02435044;-0.35454585;1.73304027;1.38327166;0.33709665;-1.07568816;0.35512227;1.87887447;0.73001908;0.0153715;-0.08519375;0.34442849\"}\n",
      "{\"tag_0\": \"0.52494651;2.30750074;1.52592016;-1.28765914;-0.81591585;1.27480322;1.80023474;0.12813667;-0.9588904;0.22382779;2.22792607;1.68425286;-0.618425;-0.87683118;0.8990627;1.88112075;1.61991296;-1.22260502;-0.93997002;1.28694876;2.60548031;4.3624207;24.94644676;-1.24718333;17.35034885;-20.55806175;-4.59469517;-23.94462942;18.12253272;-21.64123494\", \"tag_1\": \"7.97585911e-01;1.99396499e+00;-7.17155657e-01;1.15317128e-01;1.64951952e+00;1.55436616e-01;-2.49192541e-02;1.81433677e+00;6.71708341e-01;-4.60809782e-01;1.55187570e+00;3.89952555e-01;-4.72604195e-01;1.36736002e+00;5.37757951e-01;-4.24087084e-01;9.20343717e-01;1.20216374e+00;-7.02039245e-01;5.39241649e-01;1.68714278e+00;5.28573827e+00;-2.97477523e+00;-1.57829490e+01;1.65446039e-02;5.86526980e+00;2.35283996e+01;1.89037649e+00;7.06652452e+00;2.07759464e+01\", \"tag_2\": \"0.0467342;1.77271942;1.34497244;0.57707641;-0.52586821;-0.5344086;0.86836256;2.23263538;1.09425382;-0.4751886;-1.35479425;0.22175995;1.13938778;2.17824549;0.19398628;-0.92989908;-1.06901658;0.49728079;2.28369749;1.39656274;0.33204352;8.71794941;1.44131819;19.65151682;-14.89923527;7.48395208;-16.43244515;-8.05880455;10.50790228;-22.7910419\", \"tag_3\": \"2.13427876e-02;2.32508471e+00;-4.59384633e-01;-4.22458958e-01;1.38039024e+00;6.99418951e-01;-1.23382263e+00;1.40739611e-01;1.63249803e+00;-6.12622905e-01;-9.77151444e-01;1.62989237e+00;1.48192450e+00;-9.10834620e-01;1.04040242e+00;2.10175371e+00;1.55115784e-01;-8.88797603e-01;1.32809313e+00;7.07054013e-01;-1.25020256e+00;-6.85292085e+00;2.87422910e+01;2.71030144e+00;-1.12449623e+01;-2.55364303e+01;-1.50690019e+01;-9.52970903e+00;-1.07568957e+01;-2.81648214e+01\", \"tag_4\": \"0.73287365;1.36180986;1.43065531;-0.10370082;-0.4443194;0.34385391;1.68280177;0.58334118;-0.19710264;-0.20602948;1.46444107;1.52896532;0.72828249;-0.2093811;-0.14278271;1.30636694;1.33975716;-0.06555973;-0.6946629;0.42337979;1.78894264;-26.2773364;-2.52555829;-17.74821318;2.91317954;24.31799318;-10.33218681;6.86935028;3.68243741;18.87684323\"}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with open('test_sequences.json', 'w') as outfile:\n",
    "  test_data_normal_string_list = unlabeled_test_mixed_sequences_array.tolist()[0:number_of_prediction_instances]\n",
    "  json_string = \"\"\n",
    "  for example in test_data_normal_string_list:\n",
    "    json_string += \"{\" + ','.join([\"{0}: \\\"{1}\\\"\".format('\\\"' + UNLABELED_CSV_COLUMNS[i] + '\\\"', example[i]) \n",
    "                                   for i in range(len(UNLABELED_CSV_COLUMNS))]) + \"}\\n\"\n",
    "  json_string = json_string.replace(' ', '').replace(':', ': ').replace(',', ', ')\n",
    "  print(json_string)\n",
    "  outfile.write(\"%s\" % json_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_FEAT_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      X_TIME_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   FEAT_ANOM_FLAGS  MAHALANOBIS_DIST_FEAT                                                                                 MAHALANOBIS_DIST_TIME                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                TIME_ANOM_FLAGS\n",
      "[[0.08630851, 0.01710703, 0.94729535, 0.39874107, 0.10912429], [1.0370493262229936, 1.9774015362229937, 1.3670660462229938, 1.406343856222994, 2.1897142462229935], [1.20743555, 0.40862645, 1.47843099, 0.31937228, 1.52916082], [1.14710842, 0.54163604, 0.16625841, 0.16407445, 0.177806], [0.48609804, 2.06510263, 0.75320741, 1.40468418, 0.54337799], [0.8358120362269602, 0.6754584437730398, 1.0890476337730397, 0.1860635862269603, 0.2308247137730397], [1.4010847412676108, 0.9205376787323891, 0.46329691126761097, 1.762765098732389, 0.484301091267611], [2.0746489378869337, 0.27051817788693366, 0.6591949278869336, 2.083667097886934, 0.9995931178869337], [1.8706072460646652, 0.5470030660646652, 0.09524977393533485, 0.9320496739353348, 1.4803744560646652], [0.19057113, 0.99214653, 0.61741024, 0.24066002, 0.08906497], [1.8352755, 1.59915954, 0.55178713, 0.20638952, 0.65940998], [0.20091220863946302, 1.048938598639463, 1.514528048639463, 0.4584096486394631, 0.33448157863946304], [0.68841143, 1.14516381, 1.09453834, 0.8453935, 0.81842511], [1.25716387, 0.92210796, 1.59920523, 0.65794037, 0.96566587], [0.38385502, 0.52287498, 0.34086948, 0.81768388, 0.60382206], [1.8501766052001798, 1.5463956347998202, 1.6607017947998202, 0.7335204552001797, 0.8865829552001796], [0.79187743, 0.70845168, 0.9924474, 0.60762906, 1.66798859], [1.04243811, 1.03833225, 0.89896643, 0.41178175, 0.31696774], [0.52800537, 0.4001544, 1.69688929, 1.97426229, 0.408903], [0.26661931081034806, 0.534946169189652, 0.4341956491896519, 0.21587907918965188, 1.750977379189652], [1.3951040233113075, 0.052134543311307624, 1.4815913766886923, 2.021157406688692, 0.36683523331130763], [0.47596221, 0.31584913, 0.96045705, 0.75421874, 1.38432466], [0.93880634, 0.00862613, 0.59540668, 2.28180555, 0.19920215], [0.4475608115986931, 0.5150066784013069, 0.4560066984013068, 0.8714401815986932, 1.586494501598693], [1.071062294987171, 0.922738555012829, 1.4006574449871712, 1.229795815012829, 0.31995905501282895], [1.74868177, 0.24728125, 0.88623454, 2.00615088, 1.13707338], [0.03435227, 1.4809888, 0.87614424, 0.92636092, 1.69781059], [0.88146609, 0.20183649, 0.54596172, 1.06403935, 0.19309917], [0.34844452, 0.16185678, 0.32117828, 0.65159031, 0.97771496], [0.5492353796756491, 1.5919982696756492, 1.009096539675649, 0.5808300096756494, 2.392957629675649]]                   [[0.08630851, 0.01710703, 0.94729535, 0.39874107, 0.10912429], [0.8155392448301317, 1.43010645, 2.04044194, 2.00116413, 1.21779374], [1.7894313223936544, 0.40862645, 1.47843099, 0.31937228, 1.52916082], [1.14710842, 0.54163604, 0.16625841, 0.16407445, 0.177806], [0.48609804, 2.06510263, 0.75320741, 1.40468418, 0.54337799], [1.079791524193661, 0.00407659, 0.41766578, 0.85744544, 0.44055714], [1.9601996391752423, 0.12560559, 1.258229, 0.96783301, 1.27923318], [0.10701992, 1.91115068, 1.52247393, 0.09800176, 1.18207574], [1.24018965, 0.08341453, 0.72566737, 1.56246727, 0.84995686], [0.19057113, 0.99214653, 0.61741024, 0.24066002, 0.08906497], [2.209094556414139, 1.59915954, 0.55178713, 0.20638952, 0.65940998], [0.11829073788143729, 0.90426161, 0.43867216, 1.49479056, 1.61871863], [0.68841143, 1.14516381, 1.09453834, 0.8453935, 0.81842511], [1.25716387, 0.92210796, 1.59920523, 0.65794037, 0.96566587], [0.546998485641752, 0.52287498, 0.34086948, 0.81768388, 0.60382206], [2.6726128488355836, 0.74294574, 0.8572519, 1.53697035, 1.69003285], [1.2633228578640106, 0.70845168, 0.9924474, 0.60762906, 1.66798859], [1.04243811, 1.03833225, 0.89896643, 0.41178175, 0.31696774], [0.52800537, 0.4001544, 1.69688929, 1.97426229, 0.408903], [0.7203605110378788, 0.95978758, 1.0605381, 1.27885467, 0.25624363], [2.6102366348797776, 1.08263277, 0.45109315, 0.99065918, 1.39733346], [0.6409363738302718, 0.31584913, 0.96045705, 0.75421874, 1.38432466], [0.93880634, 0.00862613, 0.59540668, 2.28180555, 0.19920215], [0.34620397, 1.30877146, 1.24977148, 0.0776754, 0.79272972], [1.6828496033670979, 0.22048323, 2.10291277, 0.52754049, 0.38229627], [2.3973152466219894, 0.24728125, 0.88623454, 2.00615088, 1.13707338], [0.03435227, 1.4809888, 0.87614424, 0.92636092, 1.69781059], [0.88146609, 0.20183649, 0.54596172, 1.06403935, 0.19309917], [0.26713336835222384, 0.16185678, 0.32117828, 0.65159031, 0.97771496], [0.20852173036945265, 1.24482918, 1.82773091, 2.25599744, 0.44386982]]                                0                [5.278181568652961, 6.141749620147264, 4.815927683539757, 5.089041746915868, 5.677466535648744]       [2.3751035672303, 2.4684208075247427, 2.4180533655673746, 2.153460731526958, 2.3692281757491265, 1.6513986825098192, 1.9363004733331242, 2.743003013865368, 1.575336037417093, 2.011226491249873, 2.818959215962267, 2.6224600739035733, 0.708335584485961, 1.2910831867692614, 1.3903026251072155, 2.526718429116558, 1.8663804977688045, 1.3446901790024695, 2.364896326239051, 1.199376716021345, 2.436092504991813, 1.9574971743479157, 2.8218179630980607, 1.9177425624798694, 2.707624844029138, 2.489319856382165, 2.615486867791875, 1.6107108394962755, 2.1807091076934806, 2.738790101677753]              0\n",
      "[[0.33326098, 0.69319743, 0.01836919, 0.34392828, 0.14320316], [1.5553698962229938, 1.4100840462229938, 1.306711246222994, 1.1872854462229938, 2.1098048362229935], [0.9367061, 0.34534823, 1.45575831, 0.62486174, 1.07947085], [0.70338119, 0.18763758, 0.4292372, 0.64590716, 0.1310581], [1.04182581, 1.98480288, 1.18797205, 1.6803571, 0.3940327], [1.0930094162269604, 0.6254087237730397, 1.4715964637730399, 0.08673781622696031, 0.5597198062269603], [1.574786801267611, 1.4031556787323891, 0.09455411126761093, 2.056228498732389, 0.4573143812676109], [2.157022067886934, 0.10946190788693366, 0.42380679788693376, 1.2955331878869336, 1.3450390378869337], [1.9700252360646653, 0.04101953606466524, 0.8381195339353349, 0.9277408139353349, 1.5791972160646652], [0.03777979, 0.50059845, 0.02519642, 0.2753162, 0.00280088], [2.43879841, 1.30255595, 0.9383983, 0.69940586, 0.60816257], [0.120133121360537, 1.655022538639463, 1.831442438639463, 0.05523902863946306, 0.784482458639463], [0.63452483, 0.66609921, 1.4568485, 1.53808388, 0.67355164], [0.80411878, 1.75415711, 1.43912591, 1.39588061, 0.33326375], [0.68186144, 1.18993307, 0.9491389, 0.46195093, 0.16736942], [1.2932512352001797, 1.6292616547998202, 2.0036336247998205, 1.2958003152001798, 0.018637374799820305], [0.71299439, 0.65695489, 0.35509279, 0.23745644, 1.17160361], [1.27104652, 1.58478922, 0.95545257, 0.79142007, 0.34967432], [0.93329058, 1.12820328, 2.21985516, 1.28753278, 0.97550557], [0.27611298081034796, 1.116351469189652, 0.2883079108103481, 0.731661829189652, 0.965842989189652], [0.7203142333113075, 0.3877390233113076, 1.1643160966886925, 1.4843482766886924, 0.6064023533113077], [0.12435358, 0.80229066, 0.64774564, 0.92661575, 1.67796765], [1.22616821, 0.10926555, 0.23528506, 1.42590031, 0.1170558], [0.9341514715986932, 0.7608527584013068, 0.5038297184013069, 1.4845022615986931, 1.749036941598693], [1.234061474987171, 1.106811355012829, 1.505803674987171, 1.559161195012829, 0.20024403501282895], [1.57056515, 0.35484316, 0.63649609, 1.28799933, 1.89823695], [0.93900226, 1.67598159, 0.68651097, 0.49766533, 1.12541774], [1.15834962, 0.20869392, 0.76814667, 0.98356903, 0.27771255], [1.15378587, 0.4338172, 0.51465272, 0.32941458, 0.11889077], [0.9292507896756492, 1.1636465196756491, 1.295006849675649, 0.9646998996756491, 2.2929449696756494]]                        [[0.5911888195198072, 0.69319743, 0.01836919, 0.34392828, 0.14320316], [1.3562658673970232, 1.99742394, 2.10079674, 2.22022254, 1.29770315], [1.3819161980134558, 0.34534823, 1.45575831, 0.62486174, 1.07947085], [0.70338119, 0.18763758, 0.4292372, 0.64590716, 0.1310581], [1.04182581, 1.98480288, 1.18797205, 1.6803571, 0.3940327], [1.5261157612485046, 0.04597313, 0.80021461, 0.75811967, 1.23110166], [2.440739517368011, 0.60822359, 0.8894862, 1.26129641, 1.25224647], [0.02464679, 2.07220695, 1.75786206, 0.88613567, 0.83662982], [1.33960764, 0.58939806, 1.46853713, 1.55815841, 0.94877962], [0.2607425321705052, 0.50059845, 0.02519642, 0.2753162, 0.00280088], [3.042340978488967, 1.30255595, 0.9383983, 0.69940586, 0.60816257], [0.9717998435058139, 0.29817767, 0.12175777, 1.89796118, 1.16871775], [0.63452483, 0.66609921, 1.4568485, 1.53808388, 0.67355164], [0.80411878, 1.75415711, 1.43912591, 1.39588061, 0.33326375], [0.633255043579148, 1.18993307, 0.9491389, 0.46195093, 0.16736942], [2.0261494699935994, 0.82581176, 1.20018373, 2.09925021, 0.78481252], [1.2677425337518333, 0.65695489, 0.35509279, 0.23745644, 1.17160361], [1.27104652, 1.58478922, 0.95545257, 0.79142007, 0.34967432], [0.93329058, 1.12820328, 2.21985516, 1.28753278, 0.97550557], [0.992760566795895, 0.37838228, 1.78304166, 0.76307192, 0.52889076], [1.7401532530488077, 1.41823725, 0.13381787, 0.45385005, 1.63690058], [0.12435358, 0.80229066, 0.64774564, 0.92661575, 1.67796765], [1.22616821, 0.10926555, 0.23528506, 1.42590031, 0.1170558], [0.14038669, 1.55461754, 1.2975945, 0.69073748, 0.95527216], [1.8445546423829782, 0.40455603, 2.208059, 0.85690587, 0.50201129], [2.260403101852682, 0.35484316, 0.63649609, 1.28799933, 1.89823695], [0.93900226, 1.67598159, 0.68651097, 0.49766533, 1.12541774], [1.15834962, 0.20869392, 0.76814667, 0.98356903, 0.27771255], [1.4973538903359351, 0.4338172, 0.51465272, 0.32941458, 0.11889077], [0.3275413506500964, 1.67318093, 1.5418206, 1.87212755, 0.54388248]]                               0                [5.6309280955219565, 4.652225258273951, 5.114294222200111, 5.945892537292913, 5.465502077104699]      [2.1921621937506264, 3.0894518044367616, 1.6431253194843187, 1.8261311450801887, 2.446829266473384, 1.7194783647027556, 1.951131354175927, 2.4616498028155576, 1.3410037614706043, 2.4850248221430116, 3.322767225197329, 2.299933313215274, 1.3989417887772042, 1.9659678201061719, 1.5441376981913755, 2.251156133056701, 1.7361999239413046, 1.7545398848629687, 2.078940021099103, 1.783510319841952, 2.5052228811832133, 2.5602849680523376, 2.205830218729411, 1.7749243520240254, 2.6333137415528602, 2.4722226998852794, 1.7344412259175104, 1.4067253536916544, 2.020638202406253, 2.2560104751691488]      0\n",
      "[[0.73445717, 0.19593637, 0.75526572, 0.10518585, 0.92380376], [0.6839689462229939, 1.6387129162229939, 1.7464397262229938, 1.4348120562229938, 2.279956286222994], [0.88211537, 0.83715153, 2.08408904, 0.05511016, 0.68680086], [1.32664507, 0.59350303, 0.52117267, 0.36149535, 0.65989216], [0.5210361, 1.63039714, 0.46353711, 1.3647047, 0.65966357], [0.6235785162269604, 0.4892673237730397, 1.3315179037730398, 0.7952146462269604, 0.46366396622696027], [1.1143668312676107, 0.6151438587323891, 0.364894161267611, 1.7810407187323891, 0.9225885712676108], [1.7583750578869337, 0.05926858788693368, 0.5409160878869337, 1.4566741378869337, 1.0132892078869336], [1.4327838060646652, 0.4329031460646652, 0.6590399739353348, 1.5633357739353348, 1.3462789660646652], [0.70083891, 0.36542212, 0.19133592, 0.48821122, 0.13441075], [1.52373953, 1.57107445, 0.65200834, 0.56973544, 1.14169827], [0.11569104863946311, 0.9599890086394631, 1.517162108639463, 0.6805096086394631, 0.043353681360536855], [0.3666452, 0.53727715, 1.88170736, 0.59708671, 0.69922314], [1.5214794, 0.91906189, 2.01106892, 1.14598304, 0.6801096], [0.98290438, 0.53024939, 1.0584947, 0.17213327, 0.29903466], [1.6811604652001797, 1.5769603047998202, 1.5908735047998204, 0.9785443252001798, 0.8770819552001796], [0.96254537, 0.6343829, 0.76440265, 0.17972115, 1.2336307], [1.21225544, 1.34788263, 0.74252922, 0.81727977, 0.32646373], [0.93638252, 0.53486089, 1.94999506, 2.15765594, 1.03204426], [0.38410876918965187, 0.5022079191896519, 0.46060891081034794, 0.19097513918965192, 1.8281764991896519], [1.3191665933113077, 0.6752255433113077, 1.4553731566886925, 1.7981623866886924, 0.08775484331130756], [0.35560748, 5.34229905, 11.34676376, 4.5207422, 26.42256512], [23.20449342, 1.93771172, 8.75499856, 38.77923254, 7.4768817], [3.0098365184013067, 23.543640938401307, 12.939565968401308, 1.0520937215986932, 5.757227528401307], [26.281670055012828, 2.3966123649871713, 29.73109039498717, 11.566571324987171, 2.267756505012829], [15.92919286, 5.4089372, 16.02409734, 19.72439461, 23.40616989], [15.76944922, 22.0932672, 12.42682254, 16.23403127, 12.96872267], [14.0063463, 5.37011838, 17.83573912, 7.62880641, 0.5533737], [5.42956756, 11.2843526, 3.73843064, 4.28571629, 3.96783856], [16.91265059032435, 12.41621108032435, 14.82645111032435, 20.108022149675648, 0.18056284032435066]]    [[1.091914603748796, 0.19593637, 0.75526572, 0.10518585, 0.92380376], [0.3004032246898465, 1.76879507, 1.66106826, 1.97269593, 1.1275517], [1.1280698365130408, 0.83715153, 2.08408904, 0.05511016, 0.68680086], [1.32664507, 0.59350303, 0.52117267, 0.36149535, 0.65989216], [0.5210361, 1.63039714, 0.46353711, 1.3647047, 0.65966357], [0.37929883063531733, 0.18211453, 0.66013605, 1.4665965, 1.13504582], [1.6728826095791103, 0.17978823, 1.15982625, 0.98610863, 1.71752066], [0.4232938, 2.12240027, 1.64075277, 0.72499472, 1.16837965], [0.80236621, 0.19751445, 1.28945757, 2.19375337, 0.71586137], [0.70083891, 0.36542212, 0.19133592, 0.48821122, 0.13441075], [1.9542641249960446, 1.57107445, 0.65200834, 0.56973544, 1.14169827], [0.05608120219624424, 0.9932112, 0.4360381, 1.2726906, 1.99655389], [0.3666452, 0.53727715, 1.88170736, 0.59708671, 0.69922314], [1.5214794, 0.91906189, 2.01106892, 1.14598304, 0.6801096], [0.6503770745584211, 0.53024939, 1.0584947, 0.17213327, 0.29903466], [2.487824378221742, 0.77351041, 0.78742361, 1.78199422, 1.68053185], [1.4598262068132182, 0.6343829, 0.76440265, 0.17972115, 1.2336307], [1.21225544, 1.34788263, 0.74252922, 0.81727977, 0.32646373], [0.93638252, 0.53486089, 1.94999506, 2.15765594, 1.03204426], [0.20622392548674506, 0.99252583, 1.95534266, 1.30375861, 0.33344275], [2.5032992746366602, 1.70572377, 0.42487493, 0.76766416, 1.11825307], [0.35560748, 5.34229905, 11.34676376, 4.5207422, 26.42256512], [100.20778385573286, 1.93771172, 8.75499856, 38.77923254, 7.4768817], [3.8036013, 24.33740572, 13.73333075, 0.25832894, 6.55099231], [25.57941473, 3.09886769, 30.43334572, 12.26882665, 1.56550118], [15.92919286, 5.4089372, 16.02409734, 19.72439461, 23.40616989], [15.76944922, 22.0932672, 12.42682254, 16.23403127, 12.96872267], [14.0063463, 5.37011838, 17.83573912, 7.62880641, 0.5533737], [3.2618108611542938, 11.2843526, 3.73843064, 4.28571629, 3.96783856], [5.08173237572753, 15.25303853, 17.66327856, 17.2711947, 3.01739029]]                            0                [154.67350297193903, 115.19306019991767, 156.4848347352091, 173.43021825464592, 123.52612683072967]   [1.90256634245534, 2.5184270224429377, 2.4968160702217346, 1.343235294447726, 1.9168450767390022, 2.0993715307227925, 2.237700919093321, 2.387232228089435, 2.357109470305237, 1.9743414895560867, 2.0810893081821344, 3.1731463926782277, 2.144999070875325, 1.8596676265629375, 1.6977067483210488, 2.4820890402138085, 1.724774879504714, 1.5322914838981212, 2.558465547776955, 2.0775990225829073, 2.8779424566558798, 52.59511503772611, 153.9384622826342, 41.92836935506666, 61.580253508917885, 54.98926868311721, 50.986085024975196, 35.16624874414187, 18.605906172764538, 41.82845795738192]            1\n",
      "[[0.82922283, 0.03141424, 0.8769017, 0.74079055, 0.57473342], [1.4700895562229939, 1.5343249662229939, 1.4795522762229938, 1.4265039662229937, 2.261830066222994], [1.49492381, 0.75120973, 1.34776236, 0.42935497, 0.76228677], [0.84450981, 0.37567858, 0.2965219, 0.60234479, 0.52907114], [0.99175216, 1.79430594, 0.83390422, 1.35627275, 0.68788196], [0.5318939262269603, 0.4225388937730397, 0.8959641937730397, 0.45873990622696026, 0.3415357837730397], [1.7129002312676107, 1.149911588732389, 0.7488231312676109, 1.769983138732389, 1.2113753612676108], [1.4410416078869337, 0.1499653678869337, 0.7779709278869336, 1.8664040178869337, 1.5348976878869336], [1.6087901460646652, 0.2780074960646652, 0.8387012739353348, 1.175342723935335, 1.3259156860646653], [0.1600501, 0.12704528, 0.32129543, 0.25040035, 0.65562508], [2.07475931, 1.41145559, 1.04940981, 0.87448263, 0.65211815], [0.16534105136053712, 1.1662107586394632, 2.012411978639463, 0.48953188863946306, 0.32085476863946316], [0.8516401, 0.79337177, 1.49446773, 0.96609291, 0.27237479], [1.63834829, 1.22171682, 1.87888029, 1.08672155, 0.88844836], [0.38282412, 0.75325722, 1.08123109, 0.6199302, 0.55278754], [1.8585879652001795, 1.4419945547998203, 1.9898868347998204, 0.6836960152001796, 0.12980120520017968], [1.09576432, 0.86144481, 1.03703208, 0.48262486, 1.30141274], [0.93283521, 1.80888452, 1.10419775, 0.14518067, 0.36581009], [0.70167395, 0.57026325, 1.64290886, 1.74607215, 0.43250124], [0.11401905081034802, 1.075624789189652, 0.061739620810348006, 0.635410649189652, 1.026144639189652], [1.2399542333113076, 0.9109043533113077, 0.7150049966886924, 1.7166865566886924, 0.05344554668869239], [0.22035135, 0.54408244, 0.70443465, 0.95924103, 1.9008542], [1.34514546, 0.39212857, 0.64984686, 1.53021779, 0.14739552], [0.5353424015986932, 1.256457738401307, 0.6091257284013069, 0.8447884215986932, 1.1236717015986932], [1.140814054987171, 0.5530979350128289, 0.7101397049871712, 1.005845115012829, 0.16586831501282895], [1.61957121, 0.12620143, 0.66957773, 1.73928984, 1.03819819], [0.64942144, 2.06180843, 0.40790381, 1.28427223, 0.92842024], [1.65355856, 0.01903747, 1.20435453, 1.37103933, 0.00824748], [1.02095443, 0.65691172, 0.51918987, 0.64565625, 0.73914075], [0.5448909196756491, 1.4500078896756492, 1.066064959675649, 0.9727456496756492, 2.444375409675649]]               [[0.8295823166030665, 0.03141424, 0.8769017, 0.74079055, 0.57473342], [1.026499488717867, 1.87318302, 1.92795571, 1.98100402, 1.14567792], [1.7003178114584343, 0.75120973, 1.34776236, 0.42935497, 0.76228677], [0.84450981, 0.37567858, 0.2965219, 0.60234479, 0.52907114], [0.99175216, 1.79430594, 0.83390422, 1.35627275, 0.68788196], [1.026371597075, 0.24884296, 0.22458234, 1.13012176, 0.32984607], [2.302122617924906, 0.3549795, 1.54375522, 0.97505105, 2.00630745], [0.4421049278294948, 2.03170349, 1.40369793, 0.31526484, 0.64677117], [0.97837255, 0.3524101, 1.46911887, 1.80576032, 0.69549809], [0.1600501, 0.12704528, 0.32129543, 0.25040035, 0.65562508], [2.711634277435249, 1.41145559, 1.04940981, 0.87448263, 0.65211815], [0.23383225705389687, 0.78698945, 0.05921177, 1.46366832, 1.63234544], [0.8516401, 0.79337177, 1.49446773, 0.96609291, 0.27237479], [1.63834829, 1.22171682, 1.87888029, 1.08672155, 0.88844836], [0.37780251451413993, 0.75325722, 1.08123109, 0.6199302, 0.55278754], [2.7526712566480533, 0.63854466, 1.18643694, 1.48714591, 0.9332511], [1.5389678060588985, 0.86144481, 1.03703208, 0.48262486, 1.30141274], [0.93283521, 1.80888452, 1.10419775, 0.14518067, 0.36581009], [0.70167395, 0.57026325, 1.64290886, 1.74607215, 0.43250124], [0.5245588305849003, 0.41910896, 1.55647337, 0.8593231, 0.46858911], [1.8415072989385268, 1.94140258, 0.31549323, 0.68618833, 0.97705268], [0.375173182989951, 0.54408244, 0.70443465, 0.95924103, 1.9008542], [1.34514546, 0.39212857, 0.64984686, 1.53021779, 0.14739552], [0.25842238, 2.05022252, 1.40289051, 0.05102364, 0.32990692], [1.8743428019482569, 0.14915739, 1.41239503, 0.30358979, 0.53638701], [1.8938455025263192, 0.12620143, 0.66957773, 1.73928984, 1.03819819], [0.64942144, 2.06180843, 0.40790381, 1.28427223, 0.92842024], [1.65355856, 0.01903747, 1.20435453, 1.37103933, 0.00824748], [1.1624649369427307, 0.65691172, 0.51918987, 0.64565625, 0.73914075], [0.013788779737373247, 1.38681956, 1.77076249, 1.8640818, 0.39245204]]                  0                [4.864208629777149, 5.3099048284561245, 5.0030317496573975, 4.1663360770737405, 5.920844122587606]    [1.5272568822082804, 2.557203742002048, 1.5251748594925627, 1.4755374477403969, 1.7854692431081103, 1.715242761637805, 2.9351331733536856, 2.286278836918167, 1.7947293527623467, 2.3751855001959545, 2.865256756887478, 2.7997021639915936, 1.279936766077219, 1.7506893271402937, 1.1915050808188947, 2.50468310669284, 1.4019172959927027, 2.2450212142311434, 1.8850645934009178, 1.61069025070507, 2.663862891440416, 2.797889434815676, 1.873228379517401, 2.6870889995824174, 2.2516530650055393, 1.944324116816043, 2.413674145624164, 2.345964160369205, 0.9415013733697384, 2.4560233330235435]            0\n",
      "[[0.00395872, 0.17322761, 0.26589048, 0.19148424, 0.21639525], [0.805155476222994, 2.1466221162229937, 1.9990599962229938, 1.3386537062229937, 2.1165367162229938], [1.5949603, 0.44106297, 1.67682136, 0.29010287, 0.724104], [0.71084193, 0.08205769, 0.03902525, 0.1467994, 0.61433765], [1.33850556, 1.87909407, 0.81559788, 1.65619133, 0.00680535], [0.6408319762269602, 0.7129292137730397, 1.4205609637730396, 0.07329473377303974, 0.24955515622696023], [1.376004511267611, 0.587041598732389, 0.01469773126761087, 2.079850008732389, 1.0831233912676108], [2.156995767886934, 0.7344197678869337, 0.5928880978869338, 2.0691371878869336, 1.3793757178869337], [1.691647446064665, 0.16925167606466518, 0.8552055739353348, 1.0030079239353349, 0.7726922460646652], [0.59989862, 0.5750887, 0.19949458, 0.40367216, 0.10884144], [2.18405289, 1.11220246, 0.81230424, 0.14003887, 0.92188529], [0.4252108686394631, 0.9400613086394631, 2.255179788639463, 0.16561822863946318, 0.10205745136053701], [0.00668257, 0.66804451, 1.93347315, 1.40968721, 0.65663376], [1.51419615, 1.75556884, 2.21363113, 0.98866213, 0.2425855], [0.38615404, 0.62338149, 0.28217837, 1.05421333, 0.24612095], [1.0789717452001797, 1.1957076447998203, 1.8517515947998202, 0.9429772252001798, 0.1548170052001797], [1.67594433, 0.6553524, 0.93107819, 0.4191681, 1.15462945], [1.16729733, 0.87031166, 0.36755138, 0.62040339, 0.26062798], [0.51237519, 1.02781636, 1.44745005, 1.7590535, 1.01275174], [0.32105463918965205, 0.3565159091896519, 0.27973979918965197, 0.05025079918965192, 1.556207759189652], [1.4551763533113076, 0.4791125033113075, 0.9438690566886924, 2.0972173366886926, 0.6446839933113075], [0.40346433, 0.41324001, 1.09115891, 0.16977676, 1.49463349], [1.73140114, 0.09332299, 0.4442448, 1.42139252, 0.3449311], [0.6610972215986931, 0.6391360484013068, 0.14592913840130683, 0.5728926415986931, 1.5026822315986932], [1.7894668349871712, 1.3044558750128288, 1.067585274987171, 0.8976970750128289, 0.44444712501282896], [2.0067257, 0.17632098, 0.4989453, 2.20376187, 1.50897842], [0.86826965, 1.73060641, 0.61616411, 1.27066737, 1.26550938], [1.45925123, 0.11357086, 1.05891473, 0.82935885, 0.07863006], [0.56729308, 0.29357654, 0.59688692, 1.16824215, 0.69447811], [0.47710575967564894, 1.1152834396756492, 1.3632187396756492, 0.6066668196756493, 1.8347325996756492]]                [[0.13939114038590947, 0.17322761, 0.26589048, 0.19148424, 0.21639525], [0.07733887630487146, 1.26088587, 1.40844799, 2.06885428, 1.29097127], [1.8344056832507947, 0.44106297, 1.67682136, 0.29010287, 0.724104], [0.71084193, 0.08205769, 0.03902525, 0.1467994, 0.61433765], [1.33850556, 1.87909407, 0.81559788, 1.65619133, 0.00680535], [0.953458178261567, 0.04154736, 0.74917911, 0.59808712, 0.92093701], [1.8520173792524863, 0.20789049, 0.80962982, 1.28491792, 1.87805548], [0.02467309, 1.44724909, 1.58878076, 0.11253167, 0.80229314], [1.06122985, 0.46116592, 1.48562317, 1.63342552, 0.14227465], [0.59989862, 0.5750887, 0.19949458, 0.40367216, 0.10884144], [2.840668879117995, 1.11220246, 0.81230424, 0.14003887, 0.92188529], [0.35325365262901, 1.0131389, 0.30197958, 1.78758198, 2.05525766], [0.00668257, 0.66804451, 1.93347315, 1.40968721, 0.65663376], [1.51419615, 1.75556884, 2.21363113, 0.98866213, 0.2425855], [0.5414297382406313, 0.62338149, 0.28217837, 1.05421333, 0.24612095], [1.6879155366682395, 0.39225775, 1.0483017, 1.74642712, 0.9582669], [2.2270470865854755, 0.6553524, 0.93107819, 0.4191681, 1.15462945], [1.16729733, 0.87031166, 0.36755138, 0.62040339, 0.26062798], [0.51237519, 1.02781636, 1.44745005, 1.7590535, 1.01275174], [0.2327361259657661, 1.13821784, 1.21499395, 1.44448295, 0.06147401], [2.152755179102083, 1.50961073, 0.08662917, 1.06671911, 1.67518222], [0.6533472301435277, 0.41324001, 1.09115891, 0.16977676, 1.49463349], [1.73140114, 0.09332299, 0.4442448, 1.42139252, 0.3449311], [0.13266756, 1.43290083, 0.93969392, 0.22087214, 0.70891745], [2.3484607680225285, 0.60220055, 1.7698406, 0.19544175, 0.2578082], [1.6189380403167979, 0.17632098, 0.4989453, 2.20376187, 1.50897842], [0.86826965, 1.73060641, 0.61616411, 1.27066737, 1.26550938], [1.45925123, 0.11357086, 1.05891473, 0.82935885, 0.07863006], [0.6079988873800015, 0.29357654, 0.59688692, 1.16824215, 0.69447811], [0.46739773903394854, 1.72154401, 1.47360871, 2.23016063, 1.00209485]]                             0                [6.225756390769694, 5.4634702880591925, 5.742973892375858, 6.814974400659828, 4.680048607747062]      [2.444776491741709, 2.599998974783163, 2.2138312265770206, 2.323339664897131, 3.0004940497602335, 1.6374694183037273, 2.3781635703323145, 2.3569921741062454, 1.939420912289591, 2.00014658267055, 3.051504834067907, 3.2468100359267087, 2.357554493663671, 2.8276308792797327, 1.665530765887171, 1.5825045525652284, 1.9462863547970797, 1.5994234813438775, 1.691453991065602, 1.8829290324408305, 2.711421230220501, 2.497997369960807, 2.042953863400488, 1.9147374491512188, 3.0497718614180664, 2.6093822826750186, 1.8215191469383942, 1.9145938637216542, 1.3540529908926202, 2.56580236441579]            0\n",
      "[[0.04198157, 0.5445968, 0.74713393, 0.41999368, 0.07684221], [1.0028610562229936, 1.4689525562229937, 1.7640784362229938, 1.052320096222994, 1.9682325362229938], [0.83422681, 0.79309737, 1.87344957, 0.25778947, 0.7009711], [1.28655387, 0.19743954, 0.64392896, 0.80550182, 0.67108844], [0.4486482, 1.62495129, 0.60540843, 1.26143821, 0.27681638], [0.23786647622696033, 0.4542915537730397, 0.9321728637730398, 0.46063744622696035, 0.10152597622696025], [1.2142636012676107, 1.519007148732389, 0.19351870873238908, 1.336892628732389, 0.8453829912676109], [1.8298972478869335, 0.49327437788693373, 0.23154564788693355, 1.4444197778869337, 1.4609697578869336], [1.8223686760646651, 0.4683221960646652, 0.1740293939353348, 1.1352925639353348, 1.2007105960646651], [0.07913664, 0.05562843, 0.45761676, 0.08080697, 0.05605199], [1.69191395, 1.51895767, 0.91015551, 0.646462, 0.57439664], [0.31785744136053684, 1.641751118639463, 1.812566478639463, 0.013503371360536809, 0.6333188586394631], [0.82989558, 0.80047731, 1.79404715, 1.00162974, 0.31988395], [1.32748633, 1.48580891, 2.14000609, 0.45770954, 1.06125243], [0.96047968, 0.69011396, 0.09987011, 0.105134, 0.5133299], [1.2680159452001798, 1.1068375147998204, 1.7714454947998202, 1.5196524252001795, 0.3667721352001796], [1.30113328, 1.2909128, 0.89017087, 0.19204034, 1.3979644], [1.13774912, 1.59033414, 0.87491112, 0.37593684, 0.15867121], [0.92650173, 0.58210036, 1.96371061, 2.18621105, 0.3304164], [0.525767679189652, 0.522441789189652, 0.3618692808103481, 0.20429264918965195, 1.392762089189652], [1.2174520733113074, 0.24040190331130762, 0.5953859266886924, 1.9276037066886924, 0.7361776433113076], [2.99136024, 1.88694647, 20.41215846, 3.32734437, 22.26722639], [10.37769835, 8.8323189, 7.96100284, 37.6425341, 4.35478848], [0.8863401915986932, 34.617464141598695, 11.949028131598693, 4.357240951598693, 12.660595661598693], [27.34745520498717, 8.676049844987173, 26.172763845012827, 6.7232657049871705, 0.9329014650128289], [16.90729127, 0.65613327, 16.73292722, 21.61633275, 10.9364861], [13.01427036, 24.39548948, 1.65518979, 15.20761755, 10.47263779], [19.99435482, 6.06445066, 18.29737582, 16.44548885, 3.58262026], [4.82956485, 4.18754522, 15.43341168, 17.51024662, 2.89953151], [32.83974883967565, 17.172551710324353, 17.41468880967565, 21.06866112967565, 12.661640969675648]]      [[0.04198157, 0.5445968, 0.74713393, 0.41999368, 0.07684221], [0.938310674867797, 1.93855543, 1.64342955, 2.35518789, 1.43927545], [1.1582193781690417, 0.79309737, 1.87344957, 0.25778947, 0.7009711], [1.28655387, 0.19743954, 0.64392896, 0.80550182, 0.67108844], [0.4486482, 1.62495129, 0.60540843, 1.26143821, 0.27681638], [0.6780040176537196, 0.2170903, 0.26079101, 1.1320193, 0.77290783], [2.2654678262220282, 0.72407506, 0.60141338, 0.54196054, 1.64031508], [0.35177161, 1.68839448, 1.95012321, 0.73724908, 0.7206991], [1.19195108, 0.1620954, 0.80444699, 1.76571016, 0.570293], [0.4133144731769345, 0.05562843, 0.45761676, 0.08080697, 0.05605199], [2.2143304536529507, 1.51895767, 0.91015551, 0.646462, 0.57439664], [1.0293288461161074, 0.31144909, 0.14063373, 1.96670358, 1.31988135], [0.82989558, 0.80047731, 1.79404715, 1.00162974, 0.31988395], [1.32748633, 1.48580891, 2.14000609, 0.45770954, 1.06125243], [1.477357408058949, 0.69011396, 0.09987011, 0.105134, 0.5133299], [1.8548811054602519, 0.30338762, 0.9679956, 2.32310232, 1.17022203], [1.7765793792083648, 1.2909128, 0.89017087, 0.19204034, 1.3979644], [1.13774912, 1.59033414, 0.87491112, 0.37593684, 0.15867121], [0.92650173, 0.58210036, 1.96371061, 2.18621105, 0.3304164], [0.31916337935885253, 0.97229196, 1.85660303, 1.2904411, 0.10197166], [1.709769261734782, 1.27090013, 0.4351123, 0.89710548, 1.76667587], [2.99136024, 1.88694647, 20.41215846, 3.32734437, 22.26722639], [10.37769835, 8.8323189, 7.96100284, 37.6425341, 4.35478848], [52.18311399704431, 33.82369936, 11.15526335, 3.56347617, 11.86683088], [33.22088152539183, 9.37830517, 25.47050852, 7.42552103, 0.23064614], [14.156547716586086, 0.65613327, 16.73292722, 21.61633275, 10.9364861], [13.01427036, 24.39548948, 1.65518979, 15.20761755, 10.47263779], [19.99435482, 6.06445066, 18.29737582, 16.44548885, 3.58262026], [4.82956485, 4.18754522, 15.43341168, 17.51024662, 2.89953151], [30.00292139, 20.00937916, 14.57786136, 18.23183368, 9.82481352]]                              0                [178.13759737512416, 149.91735174937935, 161.8925004706196, 183.5146968141651, 95.44149145343397]     [2.0051853557241186, 2.9600393920595964, 2.0192634029528898, 1.2039606251605484, 2.0272804624993808, 1.6544242133514186, 2.248548984826043, 2.1056047603642396, 1.7415818341387348, 2.450027710843876, 2.441231847963209, 2.42397693553664, 1.5948439137491661, 2.3552517257412373, 2.138370016859156, 2.466374480123258, 2.0630622688463234, 2.136538617042197, 2.676191238632329, 2.066076324876469, 2.101435914536947, 53.150199423902514, 61.85338435104238, 97.01160496204488, 64.68111442700942, 45.59137682473785, 50.09154846798021, 45.47244595552717, 34.47368193907769, 62.417162304119564]               1\n",
      "[[0.26843148, 0.0129413, 0.84167842, 0.51668829, 0.71816782], [1.1471681162229936, 2.154933796222994, 1.9971476062229938, 1.9182726362229938, 1.4747157462229938], [1.46649872, 0.63354585, 1.8918916, 0.4130304, 1.1308937], [1.18559026, 0.14772502, 0.45974852, 0.8569307, 0.76595683], [1.01489562, 1.98985502, 0.7343536, 1.67864363, 0.06663918], [0.6714660062269603, 0.4633255037730397, 0.6667765437730397, 0.29018832622696034, 0.5698453162269602], [1.023995161267611, 0.9505726487323891, 0.7610637212676109, 1.2445468187323891, 1.1242658712676108], [1.5571179278869338, 0.15607836788693374, 0.01847230788693377, 1.2242639278869336, 1.7336666778869336], [1.7246290460646652, 0.0021459339353347984, 0.8990631539353349, 0.8831557439353348, 0.7510228860646652], [0.22209055, 0.60072758, 0.27410036, 0.52634693, 0.06574122], [2.135944, 1.64876388, 1.19466723, 0.48855733, 0.98898873], [0.3147691386394631, 1.238345238639463, 1.690974618639463, 0.12352222863946305, 0.568019398639463], [0.54887201, 0.61395614, 1.44656176, 0.64401769, 0.17085414], [1.19015976, 1.57488881, 1.54485897, 0.88344649, 0.23837417], [1.02346538, 1.39285713, 0.93423596, 0.43085387, 0.0814766], [1.5609098452001795, 1.8052509947998203, 1.4822351347998204, 0.7722767952001797, 0.22590803520017966], [1.15563113, 0.76538498, 0.47525442, 0.28971486, 1.43180106], [1.33679828, 0.85770778, 0.56632129, 0.68328878, 0.65021482], [1.19139794, 0.29570332, 1.41027762, 1.90457909, 0.41917392], [0.10001129918965201, 0.8977360991896519, 0.5241814291896519, 0.40637813918965193, 1.837489269189652], [1.2582090233113075, 0.26525491331130757, 0.5426313366886923, 1.4979445366886923, 0.0979652166886924], [0.47593012, 0.42318787, 1.20517916, 0.624297, 1.16802359], [1.64843334, 0.59362552, 0.02615132, 1.44398575, 0.39598329], [0.6149769015986931, 0.479407598401307, 0.02354315840130683, 1.1600809815986932, 1.4508592415986932], [0.9894943849871711, 0.861878025012829, 0.7514724349871711, 1.448008555012829, 0.8296886850128289], [2.24072223, 0.1570993, 0.77646932, 2.149788, 1.40998721], [0.35276955, 1.28972811, 0.16390073, 1.37136336, 1.29330737], [0.91466379, 0.1626878, 0.3935256, 0.48076731, 0.25137872], [0.42032311, 0.04428588, 0.80687563, 0.2749356, 0.2269882], [0.6123049696756491, 1.3811560996756491, 1.5867919596756492, 1.218910119675649, 2.7437864996756494]]                      [[0.310311786605963, 0.0129413, 0.84167842, 0.51668829, 0.71816782], [0.4922918647072603, 1.25257419, 1.41036038, 1.48923535, 1.93279224], [1.7901795514917083, 0.63354585, 1.8918916, 0.4130304, 1.1308937], [1.18559026, 0.14772502, 0.45974852, 0.8569307, 0.76595683], [1.01489562, 1.98985502, 0.7343536, 1.67864363, 0.06663918], [0.5646830367829294, 0.20805635, 0.00460531, 0.96157018, 1.24122717], [2.0188506331957177, 0.15564056, 1.55599581, 0.44961473, 1.91919796], [0.62455093, 2.02559049, 2.16319655, 0.95740493, 0.44800218], [1.09421145, 0.63256353, 1.52948075, 1.51357334, 0.12060529], [0.22209055, 0.60072758, 0.27410036, 0.52634693, 0.06574122], [2.903808301151583, 1.64876388, 1.19466723, 0.48855733, 0.98898873], [0.15363362754746412, 0.71485497, 0.26222559, 1.82967798, 1.38518081], [0.54887201, 0.61395614, 1.44656176, 0.64401769, 0.17085414], [1.19015976, 1.57488881, 1.54485897, 0.88344649, 0.23837417], [0.6303287324073645, 1.39285713, 0.93423596, 0.43085387, 0.0814766], [2.2481003761120077, 1.0018011, 0.67878524, 1.57572669, 1.02935793], [1.4932021708205119, 0.76538498, 0.47525442, 0.28971486, 1.43180106], [1.33679828, 0.85770778, 0.56632129, 0.68328878, 0.65021482], [1.19139794, 0.29570332, 1.41027762, 1.90457909, 0.41917392], [0.6746227190245129, 0.59699765, 0.97055232, 1.08835561, 0.34275552], [1.8829037358641, 1.29575314, 0.48786689, 0.46744631, 0.93253301], [0.6299090057310163, 0.42318787, 1.20517916, 0.624297, 1.16802359], [1.64843334, 0.59362552, 0.02615132, 1.44398575, 0.39598329], [0.17878788, 1.27317238, 0.81730794, 0.3663162, 0.65709446], [1.6232984344722727, 0.1596227, 1.45372776, 0.74575323, 0.12743336], [2.974532966652213, 0.1570993, 0.77646932, 2.149788, 1.40998721], [0.35276955, 1.28972811, 0.16390073, 1.37136336, 1.29330737], [0.91466379, 0.1626878, 0.3935256, 0.48076731, 0.25137872], [0.3005370436835708, 0.04428588, 0.80687563, 0.2749356, 0.2269882], [0.404662514058701, 1.45567135, 1.25003549, 1.61791733, 0.09304095]]                                    0                [4.937970076552098, 5.80156777567863, 5.807176156970092, 5.394685401882404, 5.973126087605091]        [2.082747839493436, 2.6701571971386757, 2.294520219257405, 1.3439381967091963, 2.9663606278139114, 2.289450767668952, 3.121332816681878, 2.5845764147107197, 1.8368681201197468, 2.0759792785804616, 3.2603368893747957, 2.682950587179924, 1.5982913406517814, 1.9187472037022175, 1.8474554738633602, 1.9883479185510877, 1.867766404465781, 0.9942473343589205, 2.026926892397826, 1.0300097405020676, 1.9097994272941985, 1.720065137201173, 2.204850301480755, 1.6627077073452359, 2.1471901105694315, 3.246856168333571, 2.232740650266249, 1.7989776475818646, 2.2072426454977596, 2.109947612038244]         0\n",
      "[[0.89180226, 0.84318528, 0.8540975, 0.83445521, 0.85844291], [1.091265236222994, 1.5415369462229938, 1.5067430962229937, 1.8409395262229937, 2.2050874762229937], [1.42900981, 0.21976836, 1.55675081, 0.1175664, 1.22971499], [1.18392361, 0.09125206, 0.05960444, 0.24832165, 0.68325236], [1.25722046, 1.8176748, 1.12935042, 1.76970234, 0.68442464], [0.28166100622696033, 0.5894090137730397, 1.2896801837730396, 0.5868111262269603, 0.17158485377303972], [1.9654051912676107, 0.9568161787323891, 0.05052071873238906, 1.529855018732389, 0.475921301267611], [1.4836638278869336, 0.33305288788693366, 0.11249457788693373, 1.6137462278869337, 1.1512855078869337], [2.087063136064665, 0.021627446064665157, 0.9352743439353348, 1.118828293935335, 1.1124028760646651], [0.22755546, 0.4849663, 0.24322329, 0.5440741, 0.79360583], [2.39072737, 1.82883446, 0.3942758, 0.76928938, 1.11030659], [0.1549050313605369, 0.8444427986394631, 1.7425661686394631, 0.416842498639463, 0.6050296686394632], [0.8478035, 0.51875078, 1.45739488, 0.90918041, 0.13874612], [1.46539578, 1.56315002, 1.6837368, 0.50749604, 0.67617502], [0.51741038, 1.01035818, 0.14842067, 0.72920564, 0.3399162], [1.6671908252001797, 1.3068754147998203, 1.3154627747998204, 1.1001945552001797, 0.5050794952001796], [1.05393861, 0.93220307, 0.90351649, 0.49282908, 1.72673467], [1.17411598, 1.80177097, 1.08800035, 0.28137251, 0.73408344], [1.19996648, 0.89111713, 2.26679095, 1.76892769, 0.36569274], [0.5186304691896519, 0.723952009189652, 0.310252360810348, 0.04549867918965189, 1.101944529189652], [1.5374959033113078, 0.1830619333113077, 0.5716530166886924, 2.1604359766886923, 0.8657202933113075], [3.543338, 17.63090091, 4.36797955, 20.63993453, 21.87142178], [15.18411178, 1.72919318, 10.5543302, 37.50970576, 1.25477499], [4.523074801598693, 21.696693961598694, 13.616481081598693, 10.004710381598693, 1.093561378401307], [25.556791704987173, 5.625633194987171, 24.345603025012828, 11.502404155012828, 8.056611084987171], [35.6764447, 1.52999812, 23.95030157, 14.86353444, 17.98581112], [8.55514354, 21.00848186, 7.97897494, 8.10343951, 16.83071132], [22.60172941, 0.96300052, 11.24913645, 19.41943538, 0.22739148], [18.2564907, 3.85340831, 9.45412606, 4.3992938, 13.99363541], [27.56251003032435, 33.30150214967565, 23.17797854032435, 46.432393719675645, 9.324638160324351]]             [[0.1809912370761979, 0.84318528, 0.8540975, 0.83445521, 0.85844291], [0.746352590124056, 1.86597104, 1.90076489, 1.56656846, 1.20242051], [1.875432176076692, 0.21976836, 1.55675081, 0.1175664, 1.22971499], [1.18392361, 0.09125206, 0.05960444, 0.24832165, 0.68325236], [1.25722046, 1.8176748, 1.12935042, 1.76970234, 0.68442464], [0.6118071958394096, 0.08197284, 0.61829833, 1.25819298, 0.499797], [2.7263187316388295, 0.16188409, 0.74441137, 0.73492293, 1.27085339], [0.69800503, 1.84861597, 2.06917428, 0.56792263, 1.03038335], [1.45664554, 0.60879015, 1.56569194, 1.74924589, 0.48198528], [0.22755546, 0.4849663, 0.24322329, 0.5440741, 0.79360583], [2.495111455674975, 1.82883446, 0.3942758, 0.76928938, 1.11030659], [0.63989439735726, 1.10875741, 0.21063404, 1.53635771, 1.34817054], [0.8478035, 0.51875078, 1.45739488, 0.90918041, 0.13874612], [1.46539578, 1.56315002, 1.6837368, 0.50749604, 0.67617502], [0.5994882038302719, 1.01035818, 0.14842067, 0.72920564, 0.3399162], [2.3049467135135084, 0.50342552, 0.51201288, 1.90364445, 1.30852939], [1.4438798485267712, 0.93220307, 0.90351649, 0.49282908, 1.72673467], [1.17411598, 1.80177097, 1.08800035, 0.28137251, 0.73408344], [1.19996648, 0.89111713, 2.26679095, 1.76892769, 0.36569274], [0.48953951143522323, 0.77078174, 1.80498611, 1.44923507, 0.39278922], [1.961048125120476, 1.21356016, 0.45884521, 1.12993775, 1.89621852], [19.735033115699228, 17.63090091, 4.36797955, 20.63993453, 21.87142178], [15.18411178, 1.72919318, 10.5543302, 37.50970576, 1.25477499], [18.204895771576723, 20.90292918, 12.8227163, 9.2109456, 1.88732616], [41.42292129760526, 6.32788852, 23.6433477, 10.80014883, 8.75886641], [73.5412984517042, 1.52999812, 23.95030157, 14.86353444, 17.98581112], [8.55514354, 21.00848186, 7.97897494, 8.10343951, 16.83071132], [25.628831911565374, 0.96300052, 11.24913645, 19.41943538, 0.22739148], [18.2564907, 3.85340831, 9.45412606, 4.3992938, 13.99363541], [79.26268127051178, 30.4646747, 26.01480599, 43.59556627, 12.16146561]]       0                [200.44667971307385, 141.31761552384927, 154.35514623587954, 231.02021726533997, 119.40694438822418]  [1.3852506860757745, 2.278678261364768, 2.597320039605672, 2.1157034448587346, 2.1461505825137523, 1.6600065844035465, 2.5362919237308286, 2.3899801445454427, 1.8270157699614167, 1.9471515721293935, 3.029358209278843, 2.0493905061412874, 1.5467956507590939, 1.9449019364601559, 1.7737107032118107, 2.3437883075431345, 1.9665620906371477, 1.975139269188334, 2.5451626151203857, 1.8251028287418984, 2.3123217684816177, 56.278963757084966, 62.949509882756296, 48.30991563756625, 69.82714905857837, 109.97159251942259, 43.575210284301235, 48.80750151262419, 33.72805678795814, 146.2226414214591]      1\n",
      "[[0.15337954, 0.36430147, 0.15909093, 0.57958877, 0.31762637], [1.390425696222994, 1.9861079362229939, 1.2187300262229939, 1.2204909962229937, 2.027022486222994], [0.93291083, 0.15052786, 1.74357021, 0.01000531, 0.74013553], [0.79588847, 0.36975754, 0.15424528, 0.91034762, 0.08044917], [1.33391811, 1.37599528, 1.14390818, 2.00516971, 0.42999489], [0.9626976562269604, 1.2377106237730398, 1.5660009437730396, 0.4875235062269604, 0.3581762262269602], [0.992980951267611, 0.630104118732389, 0.466806491267611, 1.2886584687323892, 1.3062961812676108], [1.8910239278869336, 0.03623913788693356, 0.6576933978869337, 1.6582056078869336, 0.9824541078869335], [1.5997388960646652, 0.11530561606466516, 0.13914377393533484, 1.6503082039353347, 1.0857387560646652], [0.10825243, 0.85607982, 0.70301387, 0.17254453, 0.05713612], [2.04250757, 1.36853124, 1.1953103, 0.8618773, 0.79972361], [0.344107801360537, 1.4208270586394631, 2.251141158639463, 0.33212374863946303, 0.5072810886394632], [0.299841, 0.33560107, 1.82918778, 0.69358897, 0.10363462], [0.81875728, 1.76377169, 1.45365066, 0.90358565, 0.42901693], [0.59409408, 1.02266286, 0.66730892, 0.59307121, 0.50389135], [1.0775929252001797, 1.3231037647998203, 1.8202400147998203, 1.1032477352001797, 0.5859836852001796], [1.35365175, 0.93491768, 0.3992378, 0.02578955, 1.09777739], [0.90467911, 0.82101721, 1.13579279, 0.53776964, 0.24972303], [0.54092312, 0.4511027, 1.44196465, 1.53817706, 1.02435044], [0.02788851918965185, 0.459952749189652, 0.023303960810348023, 0.6145461791896519, 1.849279599189652], [1.4248085933113077, 0.4994539133113076, 0.8116623066886924, 1.6968313066886924, 0.7025420433113077], [0.24984035, 0.07021636, 1.10407414, 0.77500638, 1.38327166], [1.17295784, 0.38759908, 0.24003135, 1.45668506, 0.33709665], [0.5616474715986931, 1.1370645284013068, 0.41888750840130695, 0.8357410315986932, 1.8694529415986931], [0.9420770949871711, 0.7819840250128289, 0.9130967649871711, 0.7530252150128289, 0.34713305501282893], [1.951103, 0.11664255, 1.15981526, 1.91579399, 1.87887447], [0.37166115, 1.73262589, 0.57104637, 0.50196923, 0.73001908], [0.76779075, 0.163324, 1.06882048, 0.42649724, 0.0153715], [0.53672622, 0.52521025, 0.07114742, 0.72888643, 0.08519375], [0.330517689675649, 0.6783228796756493, 1.6007391696756492, 0.7823347496756492, 2.4923989596756493]]                  [[0.44915311546622194, 0.36430147, 0.15909093, 0.57958877, 0.31762637], [1.2324561756975867, 1.42140005, 2.18877796, 2.18701699, 1.3804855], [1.12710241044906, 0.15052786, 1.74357021, 0.01000531, 0.74013553], [0.79588847, 0.36975754, 0.15424528, 0.91034762, 0.08044917], [1.33391811, 1.37599528, 1.14390818, 2.00516971, 0.42999489], [1.4560202115120864, 0.56632877, 0.89461909, 1.15890536, 1.02955808], [1.827713037858741, 0.16482797, 1.26173858, 0.49372638, 2.10122827], [0.29064493, 2.14542972, 1.52397546, 0.52346325, 1.19921475], [0.9693213, 0.51511198, 0.76956137, 2.2807258, 0.45532116], [0.2330100158411494, 0.85607982, 0.70301387, 0.17254453, 0.05713612], [2.7747412434593826, 1.36853124, 1.1953103, 0.8618773, 0.79972361], [0.633068590979911, 0.53237315, 0.29794095, 1.62107646, 1.44591912], [0.299841, 0.33560107, 1.82918778, 0.69358897, 0.10363462], [0.81875728, 1.76377169, 1.45365066, 0.90358565, 0.42901693], [0.8783078021378775, 1.02266286, 0.66730892, 0.59307121, 0.50389135], [1.7438758373571643, 0.51965387, 1.01679012, 1.90669763, 1.38943358], [1.8033407584191021, 0.93491768, 0.3992378, 0.02578955, 1.09777739], [0.90467911, 0.82101721, 1.13579279, 0.53776964, 0.24972303], [0.54092312, 0.4511027, 1.44196465, 1.53817706, 1.02435044], [0.4839569366935119, 1.034781, 1.51803771, 0.88018757, 0.35454585], [2.052198372875569, 1.52995214, 0.21883592, 0.66633308, 1.73304027], [0.23454927470025672, 0.07021636, 1.10407414, 0.77500638, 1.38327166], [1.17295784, 0.38759908, 0.24003135, 1.45668506, 0.33709665], [0.23211731, 1.93082931, 1.21265229, 0.04197625, 1.07568816], [1.6958719685947639, 0.0797287, 1.61535209, 0.05076989, 0.35512227], [2.481004672044004, 0.11664255, 1.15981526, 1.91579399, 1.87887447], [0.37166115, 1.73262589, 0.57104637, 0.50196923, 0.73001908], [0.76779075, 0.163324, 1.06882048, 0.42649724, 0.0153715], [0.8580204490185914, 0.52521025, 0.07114742, 0.72888643, 0.08519375], [0.026193534706103083, 2.15850457, 1.23608828, 2.0544927, 0.34442849]]                     0                [5.430406639570856, 5.877582855308703, 6.039343641564795, 5.368825441458327, 5.948529028201674]       [1.9241908640002217, 2.8281488908754304, 2.5837355505331114, 1.9678737112493716, 2.1868500186503566, 0.7798555206127146, 3.1315051123627957, 2.535899889454147, 2.3261945187619943, 2.0203221512630325, 2.8247648837326467, 2.2103656303208146, 2.3377259669734904, 1.7545858744954834, 1.027771907972488, 1.9043042917354018, 2.1359506942630415, 1.241233791519367, 1.7928353285381193, 1.3304036535588106, 2.6598614877577056, 2.6576855816538676, 1.8321367832084017, 2.5658763209937607, 2.6786306612080746, 3.020017747876105, 1.9668034228703324, 1.9673824230607118, 2.0231884081884424, 3.125687079101843]  0\n",
      "[[0.52494651, 0.797585911, 0.0467342, 0.0213427876, 0.73287365], [1.1000072462229937, 1.4135429962229937, 1.6347885662229937, 1.0824232762229937, 2.0456981262229936], [1.52592016, 0.717155657, 1.34497244, 0.459384633, 1.43065531], [1.28765914, 0.115317128, 0.57707641, 0.422458958, 0.10370082], [0.81591585, 1.64951952, 0.52586821, 1.38039024, 0.4443194], [0.6034213662269602, 0.5159452377730397, 1.2057904537730397, 0.02803709722696024, 0.32752794377303973], [1.0053026512676109, 0.819851342832389, 0.07343047126761093, 2.028754718732389, 0.8878696812676109], [2.0535321878869337, 0.3673320878869337, 0.050966522113066404, 2.040929246886934, 1.5983276778869335], [1.5893079960646652, 0.0412907449353348, 0.46383622393533486, 1.0020804339353349, 0.8275202360646652], [0.22382779, 0.460809782, 0.4751886, 0.612622905, 0.20602948], [2.22792607, 1.5518757, 1.35479425, 0.977151444, 1.46444107], [0.2689473486394631, 1.563247653639463, 1.731440258639463, 0.323307838639463, 0.4242348886394631], [0.618425, 0.472604195, 1.13938778, 1.4819245, 0.72828249], [0.87683118, 1.36736002, 2.17824549, 0.91083462, 0.2093811], [0.8990627, 0.537757951, 0.19398628, 1.04040242, 0.14278271], [1.0776708552001797, 1.2275369787998203, 1.7333489747998203, 1.2983038152001798, 0.5029170452001797], [1.61991296, 0.920343717, 1.06901658, 0.155115784, 1.33975716], [1.22260502, 1.20216374, 0.49728079, 0.888797603, 0.06555973], [0.93997002, 0.702039245, 2.28369749, 1.32809313, 0.6946629], [0.20778498918965194, 0.9554921001896519, 0.09817100918965194, 0.7876797361896519, 1.071353959189652], [1.5749820833113075, 0.6566445533113077, 0.6984547066886924, 2.2807007866886924, 0.7584444133113075], [4.3624207, 5.28573827, 8.71794941, 6.85292085, 26.2773364], [24.94644676, 2.97477523, 1.44131819, 28.742291, 2.52555829], [2.040948111598693, 16.576713781598695, 18.85775203840131, 1.9165366584013066, 18.541977961598693], [16.64809352498717, 0.6857107211128289, 15.601490595012828, 11.947217625012827, 2.2109242149871706], [20.55806175, 5.8652698, 7.48395208, 25.5364303, 24.31799318], [4.59469517, 23.5283996, 16.43244515, 15.0690019, 10.33218681], [23.94462942, 1.89037649, 8.05880455, 9.52970903, 6.86935028], [18.12253272, 7.06652452, 10.50790228, 10.7568957, 3.68243741], [24.47806238967565, 17.93911895032435, 25.62786934967565, 31.00164884967565, 16.04001578032435]]  [[0.8896559511059977, 0.797585911, 0.0467342, 0.0213427876, 0.73287365], [0.9872889517372654, 1.99396499, 1.77271942, 2.32508471, 1.36180986], [1.8322100677608555, 0.717155657, 1.34497244, 0.459384633, 1.43065531], [1.28765914, 0.115317128, 0.57707641, 0.422458958, 0.10370082], [0.81591585, 1.64951952, 0.52586821, 1.38039024, 0.4443194], [1.136970791648658, 0.155436616, 0.5344086, 0.699418951, 0.34385391], [1.623683409497258, 0.0249192541, 0.86836256, 1.23382263, 1.68280177], [0.12813667, 1.81433677, 2.23263538, 0.140739611, 0.58334118], [0.9588904, 0.671708341, 1.09425382, 1.63249803, 0.19710264], [0.373813521831753, 0.460809782, 0.4751886, 0.612622905, 0.20602948], [2.888864727280159, 1.5518757, 1.35479425, 0.977151444, 1.46444107], [0.3903043776719395, 0.389952555, 0.22175995, 1.62989237, 1.52896532], [0.618425, 0.472604195, 1.13938778, 1.4819245, 0.72828249], [0.87683118, 1.36736002, 2.17824549, 0.91083462, 0.2093811], [0.9327418045363733, 0.537757951, 0.19398628, 1.04040242, 0.14278271], [1.710146494515842, 0.424087084, 0.92989908, 2.10175371, 1.30636694], [2.2671506105977794, 0.920343717, 1.06901658, 0.155115784, 1.33975716], [1.22260502, 1.20216374, 0.49728079, 0.888797603, 0.06555973], [0.93997002, 0.702039245, 2.28369749, 1.32809313, 0.6946629], [0.27213925669137384, 0.539241649, 1.39656274, 0.707054013, 0.42337979], [2.0029293244405943, 1.68714278, 0.33204352, 1.25020256, 1.78894264], [4.3624207, 5.28573827, 8.71794941, 6.85292085, 26.2773364], [35.19479549968171, 2.97477523, 1.44131819, 28.742291, 2.52555829], [1.24718333, 15.782949, 19.65151682, 2.71030144, 17.74821318], [40.03354898269539, 0.0165446039, 14.89923527, 11.2449623, 2.91317954], [20.55806175, 5.8652698, 7.48395208, 25.5364303, 24.31799318], [4.59469517, 23.5283996, 16.43244515, 15.0690019, 10.33218681], [23.94462942, 1.89037649, 8.05880455, 9.52970903, 6.86935028], [5.066318811598297, 7.06652452, 10.50790228, 10.7568957, 3.68243741], [21.64123494, 20.7759464, 22.7910419, 28.1648214, 18.87684323]]  0                [173.77844124597314, 103.43956149279109, 136.132686996052, 177.4159245447427, 135.0972329929139]      [2.1132123443413326, 2.9854268785290548, 1.922214505273127, 1.99071520087851, 1.9905665254789935, 1.528627904418722, 2.2480539918476716, 2.9589843265149156, 1.6146122549268556, 1.7079963041135013, 2.998692075686814, 2.654860481092899, 1.3101788457229457, 2.2887348644240757, 1.823348315632897, 2.1128056329538603, 2.3516949064470856, 1.974486181484421, 2.259025001199457, 1.6146907154094448, 2.6669892088596674, 49.64421656528926, 66.38662487609218, 46.96020147205808, 62.03982086731704, 58.20939553323342, 48.28287703970313, 36.18558562877657, 23.4648812535131, 70.65701251028459]                1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: The `gcloud ml-engine` commands have been renamed and will soon be removed. Please use `gcloud ai-platform` instead.\n",
      "WARNING: 2019-07-12 20:42:48.115560: I tensorflow/core/platform/cpu_feature_guard.cc:145] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance critical operations:  AVX2 FMA\n",
      "To enable them in non-MKL-DNN operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2019-07-12 20:42:48.128454: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:42:48.129797: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x557f71d7f980 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:42:48.129825: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "2019-07-12 20:42:48.130614: I tensorflow/core/common_runtime/process_util.cc:115] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0712 20:42:48.130964 140104147600832 deprecation.py:323] From /usr/lib/google-cloud-sdk/lib/third_party/ml_sdk/cloud/ml/prediction/frameworks/tf_prediction_lib.py:210: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "W0712 20:42:48.350475 140104147600832 deprecation.py:323] From /usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "2019-07-12 20:42:48.447136: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "model_dir=$(ls ${PWD}/trained_model/dense_unlabeled/export/exporter | tail -1)\n",
    "gcloud ml-engine local predict \\\n",
    "  --model-dir=${PWD}/trained_model/dense_unlabeled/export/exporter/${model_dir} \\\n",
    "  --json-instances=./test_sequences.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_FEAT_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             X_TIME_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             FEAT_ANOM_FLAGS  MAHALANOBIS_DIST_FEAT                                                                                 MAHALANOBIS_DIST_TIME                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               TIME_ANOM_FLAGS\n",
      "[[0.26828822935863145, 0.23324144864138122, 0.7505111794892068, 0.10169736291415915, 0.08911301337941356], [2.0158619206413686, 1.1797579713586188, 1.843657769489207, 1.7041204229141589, 1.0195564366205865], [0.8528388106413686, 0.6589749286413812, 1.2816468194892068, 0.022328572914159184, 1.3309235166205864], [1.5017051593586315, 0.2912875613586188, 0.03052576051079317, 0.4611181570858408, 0.37604330337941355], [0.8406947793586315, 1.814754151358619, 0.9499915805107932, 1.1076404729141593, 0.7416152933794136], [1.1525971506413684, 0.2544250686413812, 0.6144499505107932, 0.5604017329141592, 0.24231983662058645], [1.8414200906413685, 0.3759540686413812, 1.061444829489207, 1.2648767170858408, 1.0809958766205865], [0.24757681935863146, 1.6608022013586188, 1.325689759489207, 0.19904194708584083, 0.9838384366205863], [1.5947863893586316, 0.16693394864138122, 0.5288831994892069, 1.265423562914159, 1.0481941633794136], [0.5451678693586315, 1.2424950086413813, 0.8141944105107932, 0.05638368708584082, 0.10917233337941357], [1.4806787606413687, 1.3488110613586188, 0.7485713005107932, 0.5034332270858408, 0.46117267662058636], [1.3976912606413685, 0.6539131313586188, 0.2418879894892068, 1.1977468529141593, 1.4204813266205865], [1.0430081693586315, 1.395512288641381, 0.8977541694892068, 0.5483497929141592, 0.6201878066205864], [1.6117606093586314, 0.6717594813586187, 1.4024210594892068, 0.9549840770858409, 1.1639031733794136], [0.02925828064136854, 0.2725265013586188, 0.14408530948920684, 0.5206401729141592, 0.8020593633794136], [2.2990297606413685, 0.9932942186413812, 1.0540360705107932, 1.2399266429141593, 1.4917955466205863], [0.43728069064136854, 0.4581032013586188, 1.1892315705107932, 0.9046727670858409, 1.4697512866205864], [1.3970348493586315, 0.7879837713586189, 0.7021822594892069, 0.7088254570858408, 0.11873043662058644], [0.8826021093586315, 0.6505028786413812, 1.500105119489207, 1.677218582914159, 0.6071403033794136], [1.4067563206413687, 0.7094391013586188, 0.8637539294892069, 0.9818109629141593, 0.45448093337941353], [2.0710055106413683, 0.8322842913586188, 0.6478773205107932, 1.2877028870858407, 1.1990961566205864], [0.12136547064136854, 0.5661976086413811, 1.1572412205107931, 0.4571750329141592, 1.1860873566205865], [1.2934030793586313, 0.2417223486413812, 0.7921908505107931, 1.9847618429141591, 0.39743945337941355], [0.008392769358631424, 1.0584229813586188, 1.0529873094892068, 0.37471910708584083, 0.9909670233794136], [1.4187208806413687, 0.47083170864138124, 1.906128599489207, 0.8245841970858409, 0.18405896662058643], [1.3940850306413686, 0.4976297286413812, 0.6894503694892069, 1.709107172914159, 0.9388360766205863], [0.38894900935863147, 1.2306403213586188, 1.0729284105107932, 0.6293172129141592, 1.4995732866205864], [1.2360628293586315, 0.45218496864138125, 0.7427458905107931, 1.3610830570858408, 0.005138133379413573], [0.006152219358631472, 0.41220525864138124, 0.12439410948920682, 0.3545466029141592, 1.1759522633794135], [1.9329953306413685, 0.9944807013586188, 1.630946739489207, 1.9589537329141589, 0.24563251662058647]]    [[0.26828822935863145, 0.23324144864138122, 0.7505111794892068, 0.10169736291415915, 0.08911301337941356], [2.0158619206413686, 1.1797579713586188, 1.843657769489207, 1.7041204229141589, 1.0195564366205865], [0.8528388106413686, 0.6589749286413812, 1.2816468194892068, 0.022328572914159184, 1.3309235166205864], [1.5017051593586315, 0.2912875613586188, 0.03052576051079317, 0.4611181570858408, 0.37604330337941355], [0.8406947793586315, 1.814754151358619, 0.9499915805107932, 1.1076404729141593, 0.7416152933794136], [1.1525971506413684, 0.2544250686413812, 0.6144499505107932, 0.5604017329141592, 0.24231983662058645], [1.8414200906413685, 0.3759540686413812, 1.061444829489207, 1.2648767170858408, 1.0809958766205865], [0.24757681935863146, 1.6608022013586188, 1.325689759489207, 0.19904194708584083, 0.9838384366205863], [1.5947863893586316, 0.16693394864138122, 0.5288831994892069, 1.265423562914159, 1.0481941633794136], [0.5451678693586315, 1.2424950086413813, 0.8141944105107932, 0.05638368708584082, 0.10917233337941357], [1.4806787606413687, 1.3488110613586188, 0.7485713005107932, 0.5034332270858408, 0.46117267662058636], [1.3976912606413685, 0.6539131313586188, 0.2418879894892068, 1.1977468529141593, 1.4204813266205865], [1.0430081693586315, 1.395512288641381, 0.8977541694892068, 0.5483497929141592, 0.6201878066205864], [1.6117606093586314, 0.6717594813586187, 1.4024210594892068, 0.9549840770858409, 1.1639031733794136], [0.02925828064136854, 0.2725265013586188, 0.14408530948920684, 0.5206401729141592, 0.8020593633794136], [2.2990297606413685, 0.9932942186413812, 1.0540360705107932, 1.2399266429141593, 1.4917955466205863], [0.43728069064136854, 0.4581032013586188, 1.1892315705107932, 0.9046727670858409, 1.4697512866205864], [1.3970348493586315, 0.7879837713586189, 0.7021822594892069, 0.7088254570858408, 0.11873043662058644], [0.8826021093586315, 0.6505028786413812, 1.500105119489207, 1.677218582914159, 0.6071403033794136], [1.4067563206413687, 0.7094391013586188, 0.8637539294892069, 0.9818109629141593, 0.45448093337941353], [2.0710055106413683, 0.8322842913586188, 0.6478773205107932, 1.2877028870858407, 1.1990961566205864], [0.12136547064136854, 0.5661976086413811, 1.1572412205107931, 0.4571750329141592, 1.1860873566205865], [1.2934030793586313, 0.2417223486413812, 0.7921908505107931, 1.9847618429141591, 0.39743945337941355], [0.008392769358631424, 1.0584229813586188, 1.0529873094892068, 0.37471910708584083, 0.9909670233794136], [1.4187208806413687, 0.47083170864138124, 1.906128599489207, 0.8245841970858409, 0.18405896662058643], [1.3940850306413686, 0.4976297286413812, 0.6894503694892069, 1.709107172914159, 0.9388360766205863], [0.38894900935863147, 1.2306403213586188, 1.0729284105107932, 0.6293172129141592, 1.4995732866205864], [1.2360628293586315, 0.45218496864138125, 0.7427458905107931, 1.3610830570858408, 0.005138133379413573], [0.006152219358631472, 0.41220525864138124, 0.12439410948920682, 0.3545466029141592, 1.1759522633794135], [1.9329953306413685, 0.9944807013586188, 1.630946739489207, 1.9589537329141589, 0.24563251662058647]]    0                [5.294090330914364, 5.272883985682681, 5.066301792574516, 5.566308538688948, 6.029748054359164]       [2.300124453792875, 2.2109491145441815, 2.6835244587453566, 2.6145807680115083, 2.279918124148294, 1.743170611642068, 1.6002316538335855, 2.3811264605695057, 1.671957836177184, 2.306439470267816, 2.1790045463563543, 1.9227997999188802, 1.4871086409129475, 1.6372270164295073, 2.439513971618757, 2.4181557014813055, 2.6040375059258705, 1.759524563610788, 2.660216046117967, 0.8344796488708106, 1.7918557995497049, 2.327877253307829, 2.8643616516133785, 2.0752775638720196, 2.5089217686853105, 1.8533876950472457, 2.262009801295568, 2.05363488212877, 2.599745415717415, 2.621013030343355]          0\n",
      "[[0.021335759358631468, 0.4428489513586188, 0.17841498051079316, 0.04688457291415915, 0.055034143379413564], [1.4975413506413684, 1.7470754613586188, 1.9040125694892067, 1.923178832914159, 1.0994658466205864], [0.5821093606413685, 0.5956967086413811, 1.258974139489207, 0.9219054470858409, 0.8812335466205865], [1.0579779293586316, 0.4379860586413812, 0.23245302948920682, 0.9429508670858409, 0.32929540337941354], [1.3964225493586313, 1.7344544013586187, 1.384756220510793, 1.3833133929141592, 0.5922700033794136], [1.4097945306413684, 0.2043753486413812, 0.9969987805107932, 0.4610759629141592, 1.0328643566205864], [2.0151221506413686, 0.8585720686413812, 0.6927020294892068, 1.5583401170858409, 1.0540091666205864], [0.3299499493586315, 1.8218584713586188, 1.5610778894892068, 0.5890919629141592, 0.6383925166205864], [1.6942043793586317, 0.33904958135861873, 1.271752959489207, 1.2611147029141594, 1.1470169233794136], [0.31681694935863147, 0.7509469286413812, 0.22198059051079316, 0.5723599070858408, 0.20103818337941357], [2.0842016706413684, 1.0522074713586187, 1.1351824705107931, 0.9964495670858409, 0.40992526662058637], [1.7187365906413685, 0.047829191358618794, 0.07502640051079316, 1.6009174729141593, 0.9704804466205865], [0.9891215693586315, 0.9164476886413813, 1.2600643294892069, 1.2410401729141594, 0.4753143366205864], [1.1587155193586316, 1.5038086313586188, 1.242341739489207, 1.6929243170858408, 0.5315010533794136], [0.3272647006413686, 0.9395845913586187, 0.7523547294892068, 0.16490722291415916, 0.3656067233794136], [1.7421043906413685, 1.076160238641381, 1.3969679005107931, 1.8022065029141592, 0.5865752166205864], [0.3583976506413685, 0.40660641135861875, 0.5518769605107932, 0.5345001470858408, 0.9733663066205864], [1.6256432593586316, 1.3344407413586188, 0.7586683994892067, 1.088463777085841, 0.15143701662058642], [1.2878873193586315, 1.3785517586413811, 2.0230709894892067, 0.9904890729141592, 1.1737428733794135], [1.4162499906413686, 0.1280338013586188, 1.586257489489207, 0.4660282129141592, 0.33065345662058643], [1.3962157206413686, 1.1678887713586188, 0.33060204051079317, 0.7508937570858408, 1.4386632766205865], [0.47895031935863147, 1.0526391386413811, 0.8445298105107932, 0.6295720429141591, 1.4797303466205864], [1.5807649493586315, 0.3596140286413812, 0.43206923051079316, 1.1288566029141593, 0.3152931033794136], [0.4949834293586315, 1.3042690613586188, 1.1008103294892069, 0.9877811870858408, 1.1535094633794136], [1.5817200606413686, 0.6549045086413812, 2.011274829489207, 1.1539495770858408, 0.3037739866205864], [1.2159684106413686, 0.6051916386413811, 0.4397119194892068, 0.9909556229141593, 1.6999996466205864], [1.2935989993586314, 1.4256331113586187, 0.8832951405107932, 0.2006216229141592, 0.9271804366205865], [1.5129463593586316, 0.041654558641381206, 0.9649308405107933, 1.280612737085841, 0.07947524662058644], [0.7991891306413687, 0.6841656786413812, 0.3178685494892068, 0.03237087291415919, 0.31712807337941357], [1.5529799206413686, 1.4228324513586188, 1.345036429489207, 1.5750838429141591, 0.34564517662058636]]           [[0.021335759358631468, 0.4428489513586188, 0.17841498051079316, 0.04688457291415915, 0.055034143379413564], [1.4975413506413684, 1.7470754613586188, 1.9040125694892067, 1.923178832914159, 1.0994658466205864], [0.5821093606413685, 0.5956967086413811, 1.258974139489207, 0.9219054470858409, 0.8812335466205865], [1.0579779293586316, 0.4379860586413812, 0.23245302948920682, 0.9429508670858409, 0.32929540337941354], [1.3964225493586313, 1.7344544013586187, 1.384756220510793, 1.3833133929141592, 0.5922700033794136], [1.4097945306413684, 0.2043753486413812, 0.9969987805107932, 0.4610759629141592, 1.0328643566205864], [2.0151221506413686, 0.8585720686413812, 0.6927020294892068, 1.5583401170858409, 1.0540091666205864], [0.3299499493586315, 1.8218584713586188, 1.5610778894892068, 0.5890919629141592, 0.6383925166205864], [1.6942043793586317, 0.33904958135861873, 1.271752959489207, 1.2611147029141594, 1.1470169233794136], [0.31681694935863147, 0.7509469286413812, 0.22198059051079316, 0.5723599070858408, 0.20103818337941357], [2.0842016706413684, 1.0522074713586187, 1.1351824705107931, 0.9964495670858409, 0.40992526662058637], [1.7187365906413685, 0.047829191358618794, 0.07502640051079316, 1.6009174729141593, 0.9704804466205865], [0.9891215693586315, 0.9164476886413813, 1.2600643294892069, 1.2410401729141594, 0.4753143366205864], [1.1587155193586316, 1.5038086313586188, 1.242341739489207, 1.6929243170858408, 0.5315010533794136], [0.3272647006413686, 0.9395845913586187, 0.7523547294892068, 0.16490722291415916, 0.3656067233794136], [1.7421043906413685, 1.076160238641381, 1.3969679005107931, 1.8022065029141592, 0.5865752166205864], [0.3583976506413685, 0.40660641135861875, 0.5518769605107932, 0.5345001470858408, 0.9733663066205864], [1.6256432593586316, 1.3344407413586188, 0.7586683994892067, 1.088463777085841, 0.15143701662058642], [1.2878873193586315, 1.3785517586413811, 2.0230709894892067, 0.9904890729141592, 1.1737428733794135], [1.4162499906413686, 0.1280338013586188, 1.586257489489207, 0.4660282129141592, 0.33065345662058643], [1.3962157206413686, 1.1678887713586188, 0.33060204051079317, 0.7508937570858408, 1.4386632766205865], [0.47895031935863147, 1.0526391386413811, 0.8445298105107932, 0.6295720429141591, 1.4797303466205864], [1.5807649493586315, 0.3596140286413812, 0.43206923051079316, 1.1288566029141593, 0.3152931033794136], [0.4949834293586315, 1.3042690613586188, 1.1008103294892069, 0.9877811870858408, 1.1535094633794136], [1.5817200606413686, 0.6549045086413812, 2.011274829489207, 1.1539495770858408, 0.3037739866205864], [1.2159684106413686, 0.6051916386413811, 0.4397119194892068, 0.9909556229141593, 1.6999996466205864], [1.2935989993586314, 1.4256331113586187, 0.8832951405107932, 0.2006216229141592, 0.9271804366205865], [1.5129463593586316, 0.041654558641381206, 0.9649308405107933, 1.280612737085841, 0.07947524662058644], [0.7991891306413687, 0.6841656786413812, 0.3178685494892068, 0.03237087291415919, 0.31712807337941357], [1.5529799206413686, 1.4228324513586188, 1.345036429489207, 1.5750838429141591, 0.34564517662058636]]           0                [5.928616123864406, 4.978707375056978, 4.876340694671175, 5.427228589530377, 5.55395638514624]        [2.6120567360808273, 3.0677311194997405, 1.6409013288314143, 1.6335623711171399, 2.1302233740563397, 2.242397543327242, 1.5802074442280918, 2.5249918872411157, 1.7739738410283987, 2.1386178279451387, 2.114946860743005, 2.395897537938759, 1.384578347451653, 2.4919891971572525, 1.7209751221363092, 1.9406368423389595, 1.7424286712477497, 2.151228068770455, 2.2606976555443583, 2.743256484176932, 2.1806555630080746, 2.0418213986854807, 1.670695968388564, 1.9756493515544793, 2.393370281956191, 2.2336059011857685, 2.4201681314050796, 2.179840203637081, 2.280027959138813, 2.114143304928164]       0\n",
      "[[0.37986043064136854, 0.05441210864138121, 0.5584815494892068, 0.19185785708584083, 0.7255664566205864], [2.3689423006413683, 1.5184465913586187, 1.4642840894892069, 1.675652222914159, 0.9293143966205863], [0.5275186306413685, 1.0875000086413813, 1.8873048694892067, 0.24193354708584083, 0.48856355662058637], [1.6812418093586317, 0.34315455135861883, 0.32438849948920684, 0.6585390570858408, 0.8581294633794135], [0.8756328393586315, 1.3800486613586187, 0.6603212805107932, 1.0676609929141594, 0.8579008733794136], [0.9403636306413686, 0.06823394864138121, 0.8569202205107931, 1.1695527929141591, 0.9368085166205864], [1.5547021806413683, 0.07056024864138122, 0.963042079489207, 1.2831523370858409, 1.5192833566205863], [0.06869706064136855, 1.8720517913586188, 1.4439685994892069, 0.42795101291415916, 0.9701423466205865], [1.1569629493586313, 0.0528340286413812, 1.0926733994892068, 1.896709662914159, 0.9140986733794136], [1.0554356493586314, 0.6157705986413813, 0.38812009051079316, 0.7852549270858409, 0.33264805337941356], [1.1691427906413687, 1.3207259713586188, 0.8487925105107932, 0.8667791470858408, 0.9434609666205864], [1.4829124206413686, 0.7428627213586187, 0.23925392948920682, 0.9756468929141592, 1.7983165866205864], [0.7212419393586315, 0.7876256286413812, 1.684923189489207, 0.3000430029141592, 0.5009858366205864], [1.8760761393586316, 0.6687134113586188, 1.814284749489207, 1.4430267470858409, 0.8783469033794136], [0.6283076406413686, 0.27990091135861883, 0.8617105294892069, 0.12491043708584082, 0.10079735662058645], [2.1300136206413685, 1.023858888641381, 0.9842077805107932, 1.4849505129141591, 1.4822945466205864], [0.6079486306413685, 0.38403442135861876, 0.9611868205107932, 0.11732255708584083, 1.0353933966205864], [1.5668521793586314, 1.0975341513586188, 0.5457450494892069, 1.1143234770858408, 0.12822642662058645], [1.2909792593586316, 0.7852093686413812, 1.753210889489207, 1.8606122329141592, 1.2302815633794135], [0.7560282406413686, 0.7421773513586188, 1.7585584894892068, 1.006714902914159, 0.5316800533794136], [1.9950680806413685, 1.4553752913586189, 0.6216591005107932, 1.0647078670858408, 0.9200157666205864], [0.7102042193586314, 5.091950571358619, 11.149979589489206, 4.817785907085841, 26.224327816620587], [22.84989668064137, 2.1880601986413812, 8.951782730510795, 39.07627624708584, 7.675119003379414], [3.4490045606413684, 24.087057241358618, 13.536546579489206, 0.5553726470858409, 6.352755006620586], [25.93401146935863, 2.848519211358619, 30.236561549489206, 11.971782942914158, 1.7637384833794136], [16.28378959935863, 5.158588721358618, 15.827313169489207, 19.42735090291416, 23.207932586620586], [16.124045959358632, 21.84291872135862, 12.623606710510794, 15.936987562914158, 13.166959973379413], [14.360943039358633, 5.6204668586413815, 18.03252329051079, 7.331762702914159, 0.3551363966205864], [5.074970820641369, 11.034004121358619, 3.5416464694892067, 3.988672582914159, 3.7696012566205868], [19.39488130064137, 15.002690051358618, 17.466494389489206, 17.56823840708584, 2.8191529866205864]]                                    [[0.37986043064136854, 0.05441210864138121, 0.5584815494892068, 0.19185785708584083, 0.7255664566205864], [2.3689423006413683, 1.5184465913586187, 1.4642840894892069, 1.675652222914159, 0.9293143966205863], [0.5275186306413685, 1.0875000086413813, 1.8873048694892067, 0.24193354708584083, 0.48856355662058637], [1.6812418093586317, 0.34315455135861883, 0.32438849948920684, 0.6585390570858408, 0.8581294633794135], [0.8756328393586315, 1.3800486613586187, 0.6603212805107932, 1.0676609929141594, 0.8579008733794136], [0.9403636306413686, 0.06823394864138121, 0.8569202205107931, 1.1695527929141591, 0.9368085166205864], [1.5547021806413683, 0.07056024864138122, 0.963042079489207, 1.2831523370858409, 1.5192833566205863], [0.06869706064136855, 1.8720517913586188, 1.4439685994892069, 0.42795101291415916, 0.9701423466205865], [1.1569629493586313, 0.0528340286413812, 1.0926733994892068, 1.896709662914159, 0.9140986733794136], [1.0554356493586314, 0.6157705986413813, 0.38812009051079316, 0.7852549270858409, 0.33264805337941356], [1.1691427906413687, 1.3207259713586188, 0.8487925105107932, 0.8667791470858408, 0.9434609666205864], [1.4829124206413686, 0.7428627213586187, 0.23925392948920682, 0.9756468929141592, 1.7983165866205864], [0.7212419393586315, 0.7876256286413812, 1.684923189489207, 0.3000430029141592, 0.5009858366205864], [1.8760761393586316, 0.6687134113586188, 1.814284749489207, 1.4430267470858409, 0.8783469033794136], [0.6283076406413686, 0.27990091135861883, 0.8617105294892069, 0.12491043708584082, 0.10079735662058645], [2.1300136206413685, 1.023858888641381, 0.9842077805107932, 1.4849505129141591, 1.4822945466205864], [0.6079486306413685, 0.38403442135861876, 0.9611868205107932, 0.11732255708584083, 1.0353933966205864], [1.5668521793586314, 1.0975341513586188, 0.5457450494892069, 1.1143234770858408, 0.12822642662058645], [1.2909792593586316, 0.7852093686413812, 1.753210889489207, 1.8606122329141592, 1.2302815633794135], [0.7560282406413686, 0.7421773513586188, 1.7585584894892068, 1.006714902914159, 0.5316800533794136], [1.9950680806413685, 1.4553752913586189, 0.6216591005107932, 1.0647078670858408, 0.9200157666205864], [0.7102042193586314, 5.091950571358619, 11.149979589489206, 4.817785907085841, 26.224327816620587], [22.84989668064137, 2.1880601986413812, 8.951782730510795, 39.07627624708584, 7.675119003379414], [3.4490045606413684, 24.087057241358618, 13.536546579489206, 0.5553726470858409, 6.352755006620586], [25.93401146935863, 2.848519211358619, 30.236561549489206, 11.971782942914158, 1.7637384833794136], [16.28378959935863, 5.158588721358618, 15.827313169489207, 19.42735090291416, 23.207932586620586], [16.124045959358632, 21.84291872135862, 12.623606710510794, 15.936987562914158, 13.166959973379413], [14.360943039358633, 5.6204668586413815, 18.03252329051079, 7.331762702914159, 0.3551363966205864], [5.074970820641369, 11.034004121358619, 3.5416464694892067, 3.988672582914159, 3.7696012566205868], [19.39488130064137, 15.002690051358618, 17.466494389489206, 17.56823840708584, 2.8191529866205864]]                                    0                [137.09493772203587, 120.56504085753348, 142.14495521998603, 164.27680386860519, 134.1159213367838]   [2.165598097512677, 2.3880774891599272, 2.3577095216135167, 2.131527182127838, 1.6164450725072959, 1.9062222935569857, 2.4502443592034493, 2.8035789155633317, 2.9950841969467725, 1.3650228204648525, 1.107713590529601, 2.5920297019323697, 2.0307476452574345, 2.0584323047807938, 2.1992371180059416, 2.046297279777221, 2.1095823839126946, 2.0488625702064116, 2.8451520984507526, 2.014971560370379, 2.2701437665586592, 59.17885569146349, 75.75018575193846, 47.502650380368486, 64.85862249472216, 61.65506184383935, 54.14177898980861, 36.27065371123264, 21.356529064348262, 48.02667225113136]        1\n",
      "[[0.4746260906413685, 0.2189342386413812, 0.6801175294892068, 0.4437468429141592, 0.37649611662058646], [1.5828216906413686, 1.6228345413586187, 1.7311715394892069, 1.6839603129141594, 0.9474406166205864], [1.1403270706413684, 1.0015582086413812, 1.1509781894892068, 0.7263986770858408, 0.5640494666205864], [1.1991065493586315, 0.12533010135861877, 0.09973772948920684, 0.8993884970858408, 0.7273084433794136], [1.3463488993586314, 1.5439574613586189, 1.030688390510793, 1.059229042914159, 0.8861192633794136], [0.8486790406413686, 0.0015055186413812094, 0.42136651051079316, 0.8330780529141592, 0.13160876662058643], [2.1532355806413683, 0.6053279786413812, 1.3469710494892069, 1.2720947570858407, 1.8080701466205864], [0.3860305106413685, 1.7813550113586187, 1.206913759489207, 0.018221132914159166, 0.44853386662058636], [1.3329692893586316, 0.10206162135861879, 1.2723346994892069, 1.5087166129141591, 0.8937353933794135], [0.5146468393586314, 0.37739375864138125, 0.5180796005107932, 0.04664335708584083, 0.8538623833794136], [1.7201625706413686, 1.1611071113586189, 1.246193980510793, 1.1715263370858409, 0.45388084662058636], [1.7639445206413686, 0.5366409713586188, 0.25599594051079316, 1.166624612914159, 1.4341081366205863], [1.2062368393586316, 1.0437202486413812, 1.2976835594892069, 0.6690492029141591, 0.07413748662058642], [1.9929450293586313, 0.9713683413586187, 1.6820961194892068, 1.383765257085841, 1.0866856633794135], [0.028227380641368538, 0.5029087413586187, 0.8844469194892068, 0.32288649291415916, 0.7510248433794136], [2.3074411206413683, 0.8888931386413812, 1.3832211105107932, 1.1901022029141592, 0.7350137966205864], [0.7411675806413686, 0.6110963313586187, 1.233816250510793, 0.7796685670858408, 1.1031754366205864], [1.2874319493586315, 1.558536041358619, 0.9074135794892069, 0.44222437708584084, 0.16757278662058642], [1.0562706893586316, 0.8206117286413812, 1.4461246894892068, 1.4490284429141593, 0.6307385433794136], [1.2541560606413684, 0.16876048135861876, 1.3596891994892069, 0.5622793929141592, 0.27035180662058644], [1.9158557206413684, 1.6910541013586189, 0.11870905948920685, 0.9832320370858408, 0.7788153766205864], [0.13424538935863145, 0.7944309186413812, 0.9012188205107932, 0.6621973229141592, 1.7026168966205864], [1.6997421993586315, 0.1417800913586188, 0.8466310305107931, 1.233174082914159, 0.3456328233794136], [0.09617435935863144, 1.799874041358619, 1.2061063394892069, 0.3480673470858408, 0.5281442233794136], [1.4884726406413686, 0.10119108864138121, 1.215610859489207, 0.6006334970858409, 0.3381497066205864], [1.2649744706413686, 0.12414704864138121, 0.4727935594892068, 1.4422461329141592, 0.8399608866205863], [1.0040181793586314, 1.811459951358619, 0.6046879805107932, 0.9872285229141592, 0.7301829366205864], [2.0081552993586316, 0.2313110086413812, 1.4011387005107931, 1.6680830370858408, 0.18998982337941356], [0.6663576906413685, 0.9072601986413812, 0.3224056994892069, 0.3486125429141592, 0.9373780533794136], [1.9373397906413685, 1.1364710813586187, 1.573978319489207, 1.5670380929141592, 0.19421473662058644]]           [[0.4746260906413685, 0.2189342386413812, 0.6801175294892068, 0.4437468429141592, 0.37649611662058646], [1.5828216906413686, 1.6228345413586187, 1.7311715394892069, 1.6839603129141594, 0.9474406166205864], [1.1403270706413684, 1.0015582086413812, 1.1509781894892068, 0.7263986770858408, 0.5640494666205864], [1.1991065493586315, 0.12533010135861877, 0.09973772948920684, 0.8993884970858408, 0.7273084433794136], [1.3463488993586314, 1.5439574613586189, 1.030688390510793, 1.059229042914159, 0.8861192633794136], [0.8486790406413686, 0.0015055186413812094, 0.42136651051079316, 0.8330780529141592, 0.13160876662058643], [2.1532355806413683, 0.6053279786413812, 1.3469710494892069, 1.2720947570858407, 1.8080701466205864], [0.3860305106413685, 1.7813550113586187, 1.206913759489207, 0.018221132914159166, 0.44853386662058636], [1.3329692893586316, 0.10206162135861879, 1.2723346994892069, 1.5087166129141591, 0.8937353933794135], [0.5146468393586314, 0.37739375864138125, 0.5180796005107932, 0.04664335708584083, 0.8538623833794136], [1.7201625706413686, 1.1611071113586189, 1.246193980510793, 1.1715263370858409, 0.45388084662058636], [1.7639445206413686, 0.5366409713586188, 0.25599594051079316, 1.166624612914159, 1.4341081366205863], [1.2062368393586316, 1.0437202486413812, 1.2976835594892069, 0.6690492029141591, 0.07413748662058642], [1.9929450293586313, 0.9713683413586187, 1.6820961194892068, 1.383765257085841, 1.0866856633794135], [0.028227380641368538, 0.5029087413586187, 0.8844469194892068, 0.32288649291415916, 0.7510248433794136], [2.3074411206413683, 0.8888931386413812, 1.3832211105107932, 1.1901022029141592, 0.7350137966205864], [0.7411675806413686, 0.6110963313586187, 1.233816250510793, 0.7796685670858408, 1.1031754366205864], [1.2874319493586315, 1.558536041358619, 0.9074135794892069, 0.44222437708584084, 0.16757278662058642], [1.0562706893586316, 0.8206117286413812, 1.4461246894892068, 1.4490284429141593, 0.6307385433794136], [1.2541560606413684, 0.16876048135861876, 1.3596891994892069, 0.5622793929141592, 0.27035180662058644], [1.9158557206413684, 1.6910541013586189, 0.11870905948920685, 0.9832320370858408, 0.7788153766205864], [0.13424538935863145, 0.7944309186413812, 0.9012188205107932, 0.6621973229141592, 1.7026168966205864], [1.6997421993586315, 0.1417800913586188, 0.8466310305107931, 1.233174082914159, 0.3456328233794136], [0.09617435935863144, 1.799874041358619, 1.2061063394892069, 0.3480673470858408, 0.5281442233794136], [1.4884726406413686, 0.10119108864138121, 1.215610859489207, 0.6006334970858409, 0.3381497066205864], [1.2649744706413686, 0.12414704864138121, 0.4727935594892068, 1.4422461329141592, 0.8399608866205863], [1.0040181793586314, 1.811459951358619, 0.6046879805107932, 0.9872285229141592, 0.7301829366205864], [2.0081552993586316, 0.2313110086413812, 1.4011387005107931, 1.6680830370858408, 0.18998982337941356], [0.6663576906413685, 0.9072601986413812, 0.3224056994892069, 0.3486125429141592, 0.9373780533794136], [1.9373397906413685, 1.1364710813586187, 1.573978319489207, 1.5670380929141592, 0.19421473662058644]]           0                [5.120369392410001, 4.883839034300374, 5.400474067415636, 4.472440412186019, 5.937787435990836]       [1.7333745120417883, 2.362739988002392, 0.8032109326629945, 1.8208409187776995, 1.460771679515355, 2.089571239939009, 2.905343739069875, 2.6441698690766287, 2.1867136280237203, 2.036598706473137, 1.4221434211702935, 2.1363071837751826, 1.786245559898158, 1.9006520683493962, 1.9971063410476002, 2.1572485503851078, 1.4966701841758399, 2.5279913328353243, 1.7931799301522187, 2.2082894690082853, 3.1868381167740876, 2.903093758765004, 1.7759697843885143, 2.611410219726871, 2.29444959932268, 1.8754909262571873, 2.3427797374993267, 2.488596279116523, 1.6480806462379252, 2.2428134241098183]       0\n",
      "[[0.3506380193586314, 0.07712086864138121, 0.06910630948920682, 0.10555946708584082, 0.01815794662058645], [2.2477557706413682, 1.0105373913586189, 1.2116638194892069, 1.771810572914159, 1.0927339666205864], [1.2403635606413688, 0.6914114486413812, 1.4800371894892068, 0.5871465770858408, 0.5258666966205864], [1.0654386693586315, 0.1682907886413812, 0.23580942051079318, 0.44384310708584085, 0.8125749533794135], [1.6931022993586313, 1.6287455913586188, 1.0123820505107932, 1.359147622914159, 0.20504265337941355], [0.9576170906413685, 0.2918958386413812, 0.9459632805107931, 0.30104341291415915, 0.7226997066205864], [1.8163398606413685, 0.0424579886413812, 0.6128456494892067, 1.5819616270858408, 1.6798181766205864], [0.32992364935863144, 1.1969006113586187, 1.3919965894892068, 0.18451203708584082, 0.6040558366205864], [1.4158265893586313, 0.2108174413586188, 1.2888389994892069, 1.3363818129141594, 0.34051195337941353], [0.9544953593586315, 0.8254371786413812, 0.3962787505107932, 0.7007158670858409, 0.30707874337941354], [1.8294561506413682, 0.8618539813586188, 1.0090884105107931, 0.43708257708584086, 0.7236479866205864], [1.1733926006413684, 0.7627904213586187, 0.4987637505107932, 1.4905382729141592, 1.8570203566205865], [0.36127930935863145, 0.9183929886413812, 1.7366889794892069, 1.112643502914159, 0.45839645662058637], [1.8687928893586316, 1.5052203613586188, 2.016846959489207, 1.2857058370858407, 0.4408228033794136], [0.031557300641368524, 0.37303301135861877, 0.08539419948920685, 0.7571696229141592, 0.4443582533794136], [1.5278249006413684, 0.6426062286413812, 1.245085870510793, 1.4493834129141594, 0.7600295966205864], [1.3213475906413685, 0.40500392135861873, 1.1278623605107931, 0.7162118070858408, 0.9563921466205865], [1.5218940693586314, 0.6199631813586188, 0.17076720948920682, 0.9174470970858408, 0.06239067662058642], [0.8669719293586314, 1.2781648386413813, 1.250665879489207, 1.4620097929141593, 1.2109890433794135], [0.8190823706413685, 0.8878693613586188, 1.0182097794892069, 1.1474392429141593, 0.25971131337941356], [2.1310778406413684, 1.2592622513586187, 0.11015500051079316, 1.3637628170858407, 1.4769449166205864], [0.04886759064136853, 0.6635884886413812, 1.2879430805107932, 0.12726694708584083, 1.2963961866205864], [2.0859978793586316, 0.15702548864138122, 0.6410289705107932, 1.124348812914159, 0.5431684033794135], [0.22192917935863146, 1.1825523513586187, 0.7429097494892067, 0.07617156708584083, 0.9071547533794135], [2.1371254206413686, 0.8525490286413812, 1.5730564294892069, 0.4924854570858408, 0.05957089662058643], [1.6521289606413685, 0.07402749864138122, 0.3021611294892068, 1.9067181629141592, 1.3107411166205865], [1.2228663893586313, 1.4802579313586188, 0.8129482805107933, 0.9736236629141591, 1.0672720766205863], [1.8138479693586316, 0.13677761864138122, 1.255698900510793, 1.1264025570858407, 0.27686736337941353], [0.21269634064136855, 0.5439250186413812, 0.4001027494892068, 0.8711984429141592, 0.8927154133794136], [2.0051249506413686, 1.4711955313586187, 1.2768245394892068, 1.933116922914159, 0.8038575466205864]]    [[0.3506380193586314, 0.07712086864138121, 0.06910630948920682, 0.10555946708584082, 0.01815794662058645], [2.2477557706413682, 1.0105373913586189, 1.2116638194892069, 1.771810572914159, 1.0927339666205864], [1.2403635606413688, 0.6914114486413812, 1.4800371894892068, 0.5871465770858408, 0.5258666966205864], [1.0654386693586315, 0.1682907886413812, 0.23580942051079318, 0.44384310708584085, 0.8125749533794135], [1.6931022993586313, 1.6287455913586188, 1.0123820505107932, 1.359147622914159, 0.20504265337941355], [0.9576170906413685, 0.2918958386413812, 0.9459632805107931, 0.30104341291415915, 0.7226997066205864], [1.8163398606413685, 0.0424579886413812, 0.6128456494892067, 1.5819616270858408, 1.6798181766205864], [0.32992364935863144, 1.1969006113586187, 1.3919965894892068, 0.18451203708584082, 0.6040558366205864], [1.4158265893586313, 0.2108174413586188, 1.2888389994892069, 1.3363818129141594, 0.34051195337941353], [0.9544953593586315, 0.8254371786413812, 0.3962787505107932, 0.7007158670858409, 0.30707874337941354], [1.8294561506413682, 0.8618539813586188, 1.0090884105107931, 0.43708257708584086, 0.7236479866205864], [1.1733926006413684, 0.7627904213586187, 0.4987637505107932, 1.4905382729141592, 1.8570203566205865], [0.36127930935863145, 0.9183929886413812, 1.7366889794892069, 1.112643502914159, 0.45839645662058637], [1.8687928893586316, 1.5052203613586188, 2.016846959489207, 1.2857058370858407, 0.4408228033794136], [0.031557300641368524, 0.37303301135861877, 0.08539419948920685, 0.7571696229141592, 0.4443582533794136], [1.5278249006413684, 0.6426062286413812, 1.245085870510793, 1.4493834129141594, 0.7600295966205864], [1.3213475906413685, 0.40500392135861873, 1.1278623605107931, 0.7162118070858408, 0.9563921466205865], [1.5218940693586314, 0.6199631813586188, 0.17076720948920682, 0.9174470970858408, 0.06239067662058642], [0.8669719293586314, 1.2781648386413813, 1.250665879489207, 1.4620097929141593, 1.2109890433794135], [0.8190823706413685, 0.8878693613586188, 1.0182097794892069, 1.1474392429141593, 0.25971131337941356], [2.1310778406413684, 1.2592622513586187, 0.11015500051079316, 1.3637628170858407, 1.4769449166205864], [0.04886759064136853, 0.6635884886413812, 1.2879430805107932, 0.12726694708584083, 1.2963961866205864], [2.0859978793586316, 0.15702548864138122, 0.6410289705107932, 1.124348812914159, 0.5431684033794135], [0.22192917935863146, 1.1825523513586187, 0.7429097494892067, 0.07617156708584083, 0.9071547533794135], [2.1371254206413686, 0.8525490286413812, 1.5730564294892069, 0.4924854570858408, 0.05957089662058643], [1.6521289606413685, 0.07402749864138122, 0.3021611294892068, 1.9067181629141592, 1.3107411166205865], [1.2228663893586313, 1.4802579313586188, 0.8129482805107933, 0.9736236629141591, 1.0672720766205863], [1.8138479693586316, 0.13677761864138122, 1.255698900510793, 1.1264025570858407, 0.27686736337941353], [0.21269634064136855, 0.5439250186413812, 0.4001027494892068, 0.8711984429141592, 0.8927154133794136], [2.0051249506413686, 1.4711955313586187, 1.2768245394892068, 1.933116922914159, 0.8038575466205864]]    0                [5.9272117878028965, 5.365782879129945, 5.55740091984753, 6.3071112136618845, 5.327597970892668]      [2.7341525890267144, 1.90924904107883, 1.5836927241572891, 1.96345852542513, 2.450209212519027, 1.794528278435213, 2.8191292370613943, 1.900277815502003, 1.893724129937409, 1.466772914075385, 2.4413978042324396, 2.7732932518447484, 2.659392238347569, 2.5088739153041777, 2.6561619059000554, 1.210736297490627, 1.4531144332389088, 2.302275981445159, 2.2301205933585035, 1.611618261506799, 2.893091808349727, 2.666567755157579, 2.1818115015693165, 1.9649913533489436, 3.4618052445399132, 2.8105170121873604, 1.4914375249835359, 2.161318325209572, 2.178262103680491, 2.3163040262558376]             0\n",
      "[[0.31261516935863143, 0.2942483213586188, 0.5503497594892068, 0.12294997291415916, 0.12139509337941357], [2.0500501906413686, 1.6882069513586189, 1.446645379489207, 2.058144182914159, 1.2410381466205864], [0.47963007064136853, 1.0434458486413813, 1.676665399489207, 0.5548331770858408, 0.5027337966205864], [1.6411506093586317, 0.052908938641381215, 0.44714478948920683, 1.1025455270858409, 0.8693257433794136], [0.8032449393586314, 1.3746028113586188, 0.8021926005107931, 0.9643945029141591, 0.47505368337941356], [0.5546515906413686, 0.0332581786413812, 0.45757518051079316, 0.8349755929141592, 0.5746705266205864], [1.6545989506413683, 0.9744235386413812, 0.40462920948920683, 0.8390042470858409, 1.4420777766205863], [0.0028251293586314397, 1.4380460013586187, 1.753339039489207, 0.4402053729141592, 0.5224617966205864], [1.5465478193586315, 0.08825307864138121, 0.6076628194892069, 1.4686664529141593, 0.7685303033794136], [0.27546009935863147, 0.3059769086413812, 0.6544009305107932, 0.37785067708584086, 0.2542892933794136], [1.3373172106413684, 1.2686091913586188, 1.1069396805107932, 0.9435057070858408, 0.3761593366205864], [1.9164609106413684, 0.06110061135861877, 0.056150440510793154, 1.669659872914159, 1.1216440466205864], [1.1844923193586314, 1.0508257886413812, 1.5972629794892068, 0.7045860329141592, 0.12164664662058641], [1.6820830693586313, 1.2354604313586188, 1.9432219194892069, 0.7547532470858409, 1.2594897333794135], [0.6058829406413685, 0.4397654813586188, 0.09691406051079317, 0.19190970708584082, 0.7115672033794136], [1.7168691006413686, 0.5537360986413813, 1.1647797705107932, 2.026058612914159, 0.9719847266205863], [0.9465365406413685, 1.0405643213586189, 1.086955040510793, 0.48908404708584086, 1.1997270966205864], [1.4923458593586316, 1.3399856613586187, 0.6781269494892068, 0.6729805470858408, 0.03956609337941355], [1.2810984693586316, 0.8324488386413812, 1.7669264394892068, 1.8891673429141589, 0.5286537033794135], [0.6143693306413686, 0.7219434813586187, 1.659818859489207, 0.9933973929141592, 0.09626564337941355], [1.8933535606413683, 1.0205516513586188, 0.23832812948920684, 1.194149187085841, 1.5684385666205864], [3.3459569793586317, 2.1372949486413813, 20.21537428948921, 3.624388077085841, 22.465463693379416], [10.732295089358631, 8.58197042135862, 8.157787010510793, 37.34549039291416, 4.553025783379414], [0.4471721493586315, 34.07404783864138, 11.352047520510794, 3.860519877085841, 12.065068183379413], [27.69511379064137, 9.12795669135862, 25.66729269051079, 7.128477322914159, 0.42888344337941353], [16.55269453064137, 0.9064817486413812, 16.929711390510793, 21.31928904291416, 11.134723403379413], [13.368867099358631, 24.145141001358617, 1.458405619489207, 14.91057384291416, 10.274400486620587], [20.348951559358632, 5.814102181358619, 18.49415999051079, 16.74253255708584, 3.3843829566205867], [4.474968110641369, 3.937196741358618, 15.236627509489207, 17.21320291291416, 3.0977688133794135], [30.35751812935863, 19.75903068135862, 14.774645530510794, 18.528877387085842, 10.023050823379412]]                                     [[0.31261516935863143, 0.2942483213586188, 0.5503497594892068, 0.12294997291415916, 0.12139509337941357], [2.0500501906413686, 1.6882069513586189, 1.446645379489207, 2.058144182914159, 1.2410381466205864], [0.47963007064136853, 1.0434458486413813, 1.676665399489207, 0.5548331770858408, 0.5027337966205864], [1.6411506093586317, 0.052908938641381215, 0.44714478948920683, 1.1025455270858409, 0.8693257433794136], [0.8032449393586314, 1.3746028113586188, 0.8021926005107931, 0.9643945029141591, 0.47505368337941356], [0.5546515906413686, 0.0332581786413812, 0.45757518051079316, 0.8349755929141592, 0.5746705266205864], [1.6545989506413683, 0.9744235386413812, 0.40462920948920683, 0.8390042470858409, 1.4420777766205863], [0.0028251293586314397, 1.4380460013586187, 1.753339039489207, 0.4402053729141592, 0.5224617966205864], [1.5465478193586315, 0.08825307864138121, 0.6076628194892069, 1.4686664529141593, 0.7685303033794136], [0.27546009935863147, 0.3059769086413812, 0.6544009305107932, 0.37785067708584086, 0.2542892933794136], [1.3373172106413684, 1.2686091913586188, 1.1069396805107932, 0.9435057070858408, 0.3761593366205864], [1.9164609106413684, 0.06110061135861877, 0.056150440510793154, 1.669659872914159, 1.1216440466205864], [1.1844923193586314, 1.0508257886413812, 1.5972629794892068, 0.7045860329141592, 0.12164664662058641], [1.6820830693586313, 1.2354604313586188, 1.9432219194892069, 0.7547532470858409, 1.2594897333794135], [0.6058829406413685, 0.4397654813586188, 0.09691406051079317, 0.19190970708584082, 0.7115672033794136], [1.7168691006413686, 0.5537360986413813, 1.1647797705107932, 2.026058612914159, 0.9719847266205863], [0.9465365406413685, 1.0405643213586189, 1.086955040510793, 0.48908404708584086, 1.1997270966205864], [1.4923458593586316, 1.3399856613586187, 0.6781269494892068, 0.6729805470858408, 0.03956609337941355], [1.2810984693586316, 0.8324488386413812, 1.7669264394892068, 1.8891673429141589, 0.5286537033794135], [0.6143693306413686, 0.7219434813586187, 1.659818859489207, 0.9933973929141592, 0.09626564337941355], [1.8933535606413683, 1.0205516513586188, 0.23832812948920684, 1.194149187085841, 1.5684385666205864], [3.3459569793586317, 2.1372949486413813, 20.21537428948921, 3.624388077085841, 22.465463693379416], [10.732295089358631, 8.58197042135862, 8.157787010510793, 37.34549039291416, 4.553025783379414], [0.4471721493586315, 34.07404783864138, 11.352047520510794, 3.860519877085841, 12.065068183379413], [27.69511379064137, 9.12795669135862, 25.66729269051079, 7.128477322914159, 0.42888344337941353], [16.55269453064137, 0.9064817486413812, 16.929711390510793, 21.31928904291416, 11.134723403379413], [13.368867099358631, 24.145141001358617, 1.458405619489207, 14.91057384291416, 10.274400486620587], [20.348951559358632, 5.814102181358619, 18.49415999051079, 16.74253255708584, 3.3843829566205867], [4.474968110641369, 3.937196741358618, 15.236627509489207, 17.21320291291416, 3.0977688133794135], [30.35751812935863, 19.75903068135862, 14.774645530510794, 18.528877387085842, 10.023050823379412]]                                     0                [142.83913079498328, 147.68635667217842, 152.67439751111033, 177.0542103483277, 109.79523085056026]   [2.197947355128537, 2.7986191210084237, 1.902713177309635, 1.8131270998114875, 1.639967386252259, 1.9745842058467686, 2.152649985380074, 2.6894224709374157, 1.7750182133396926, 1.9269516194082879, 1.3436878537594277, 2.574479983442606, 1.912541495869313, 2.576679714405669, 2.045949694616348, 2.302784913144423, 1.4570020409578857, 2.5083332239048066, 2.796588625542724, 2.370957452327764, 2.486310849375717, 61.12166831342947, 82.36850826408978, 67.43978788946103, 63.398000981515324, 51.11971990502569, 56.122798801851026, 44.02724537313211, 44.85406676042976, 61.37862887421603]               1\n",
      "[[0.08616525935863145, 0.23740717864138122, 0.6448942494892069, 0.21964458291415923, 0.5199305166205864], [1.9057431306413686, 1.0022257113586188, 1.2135762094892069, 1.192191642914159, 1.7345549366205864], [1.1119019806413686, 0.8838943286413812, 1.6951074294892068, 0.7100741070858408, 0.9326563966205863], [1.5401869993586312, 0.1026234586413812, 0.26296434948920683, 1.1539744070858409, 0.9641941333794136], [1.3694923593586315, 1.7395065413586188, 0.9311377705107933, 1.3815999229141593, 0.26487648337941355], [0.9882511206413686, 0.04229212864138121, 0.19217886051079316, 0.6645264729141592, 1.0429898666205863], [1.4643305106413687, 0.40598903864138125, 1.3592116394892069, 0.7466584370858409, 1.7209606566205864], [0.2699541906413685, 1.7752420113586187, 1.9664123794892068, 0.6603612229141592, 0.24976487662058647], [1.4488081893586315, 0.3822150513586188, 1.332696579489207, 1.2165296329141593, 0.31884259337941356], [0.5766872893586314, 0.8510760586413813, 0.47088453051079315, 0.8233906370858408, 0.2639785233794136], [1.7813472606413683, 1.3984154013586187, 1.3914514005107932, 0.7856010370858408, 0.7907514266205864], [1.2838343306413686, 0.46450649135861877, 0.06544141948920684, 1.532634272914159, 1.1869435066205865], [0.9034687493586314, 0.8643046186413812, 1.249777589489207, 0.34697398291415915, 0.027383163379413572], [1.5447564993586314, 1.3245403313586188, 1.3480747994892068, 1.1804901970858408, 0.43661147337941353], [0.6688686406413685, 1.142508651358619, 0.7374517894892068, 0.1338101629141592, 0.27971390337941354], [2.0097630006413683, 1.2521495786413812, 0.8755694105107932, 1.2786829829141593, 0.8311206266205864], [0.8010343906413685, 0.5150365013586188, 0.6720385905107932, 0.5867585670858408, 1.2335637566205864], [1.6913950193586316, 0.6073593013586188, 0.3695371194892068, 0.9803324870858409, 0.4519775166205864], [1.5459946793586314, 0.5460517986413812, 1.213493449489207, 1.607535382914159, 0.6174112233794136], [1.0401257106413686, 0.34664917135861884, 0.7737681494892068, 0.7913119029141592, 0.5409928233794136], [1.9341105106413683, 1.0454046613586188, 0.29108271948920683, 0.7644900170858409, 0.7342957066205864], [0.12133338064136856, 0.6735363486413812, 1.401963330510793, 0.32725329291415917, 0.9697862866205864], [2.0030300793586315, 0.3432770413586188, 0.22293549051079317, 1.146942042914159, 0.5942205933794136], [0.17580885935863144, 1.022823901358619, 0.6205237694892067, 0.6633599070858408, 0.8553317633794136], [1.3371529706413687, 0.4099711786413812, 1.256943589489207, 1.042796937085841, 0.3256706633794135], [1.8861254906413683, 0.4074477786413812, 0.5796851494892068, 1.852744292914159, 1.2117499066205863], [0.7073662893586314, 1.0393796313586188, 0.36068490051079316, 1.0743196529141592, 1.0950700666205864], [1.2692605293586314, 0.4130362786413812, 0.5903097705107931, 0.7778110170858408, 0.44961602337941353], [0.06572637064136855, 0.2060625986413812, 0.6100914594892068, 0.022108107085840822, 0.42522550337941356], [1.8699257406413685, 1.2053228713586188, 1.0532513194892068, 1.320873622914159, 0.10519635337941356]]            [[0.08616525935863145, 0.23740717864138122, 0.6448942494892069, 0.21964458291415923, 0.5199305166205864], [1.9057431306413686, 1.0022257113586188, 1.2135762094892069, 1.192191642914159, 1.7345549366205864], [1.1119019806413686, 0.8838943286413812, 1.6951074294892068, 0.7100741070858408, 0.9326563966205863], [1.5401869993586312, 0.1026234586413812, 0.26296434948920683, 1.1539744070858409, 0.9641941333794136], [1.3694923593586315, 1.7395065413586188, 0.9311377705107933, 1.3815999229141593, 0.26487648337941355], [0.9882511206413686, 0.04229212864138121, 0.19217886051079316, 0.6645264729141592, 1.0429898666205863], [1.4643305106413687, 0.40598903864138125, 1.3592116394892069, 0.7466584370858409, 1.7209606566205864], [0.2699541906413685, 1.7752420113586187, 1.9664123794892068, 0.6603612229141592, 0.24976487662058647], [1.4488081893586315, 0.3822150513586188, 1.332696579489207, 1.2165296329141593, 0.31884259337941356], [0.5766872893586314, 0.8510760586413813, 0.47088453051079315, 0.8233906370858408, 0.2639785233794136], [1.7813472606413683, 1.3984154013586187, 1.3914514005107932, 0.7856010370858408, 0.7907514266205864], [1.2838343306413686, 0.46450649135861877, 0.06544141948920684, 1.532634272914159, 1.1869435066205865], [0.9034687493586314, 0.8643046186413812, 1.249777589489207, 0.34697398291415915, 0.027383163379413572], [1.5447564993586314, 1.3245403313586188, 1.3480747994892068, 1.1804901970858408, 0.43661147337941353], [0.6688686406413685, 1.142508651358619, 0.7374517894892068, 0.1338101629141592, 0.27971390337941354], [2.0097630006413683, 1.2521495786413812, 0.8755694105107932, 1.2786829829141593, 0.8311206266205864], [0.8010343906413685, 0.5150365013586188, 0.6720385905107932, 0.5867585670858408, 1.2335637566205864], [1.6913950193586316, 0.6073593013586188, 0.3695371194892068, 0.9803324870858409, 0.4519775166205864], [1.5459946793586314, 0.5460517986413812, 1.213493449489207, 1.607535382914159, 0.6174112233794136], [1.0401257106413686, 0.34664917135861884, 0.7737681494892068, 0.7913119029141592, 0.5409928233794136], [1.9341105106413683, 1.0454046613586188, 0.29108271948920683, 0.7644900170858409, 0.7342957066205864], [0.12133338064136856, 0.6735363486413812, 1.401963330510793, 0.32725329291415917, 0.9697862866205864], [2.0030300793586315, 0.3432770413586188, 0.22293549051079317, 1.146942042914159, 0.5942205933794136], [0.17580885935863144, 1.022823901358619, 0.6205237694892067, 0.6633599070858408, 0.8553317633794136], [1.3371529706413687, 0.4099711786413812, 1.256943589489207, 1.042796937085841, 0.3256706633794135], [1.8861254906413683, 0.4074477786413812, 0.5796851494892068, 1.852744292914159, 1.2117499066205863], [0.7073662893586314, 1.0393796313586188, 0.36068490051079316, 1.0743196529141592, 1.0950700666205864], [1.2692605293586314, 0.4130362786413812, 0.5903097705107931, 0.7778110170858408, 0.44961602337941353], [0.06572637064136855, 0.2060625986413812, 0.6100914594892068, 0.022108107085840822, 0.42522550337941356], [1.8699257406413685, 1.2053228713586188, 1.0532513194892068, 1.320873622914159, 0.10519635337941356]]            0                [4.743688724302218, 5.2117454748193275, 5.757737247486373, 5.445439004734566, 5.847545684148308]      [2.116481694829235, 2.3706487558379474, 1.6499806463717275, 1.849573975632831, 2.560664043617699, 2.0419902198210544, 2.8396099982337426, 3.0873576668267035, 1.6415299753576587, 1.7204392716276562, 1.9982399524165304, 2.2912587728029292, 1.9748738211088894, 1.5172499710997456, 1.9745576477851527, 1.6915408372951009, 1.5463770897337132, 1.7005479149570428, 1.5825405868762923, 1.0273579898054683, 2.4858103682481465, 2.2550303440934583, 2.1735148550767818, 1.8823393315327, 1.4407884929319137, 2.1570759219479343, 1.8578661446146432, 1.2160130469710324, 2.3056380380237593, 2.1552819053679384]  0\n",
      "[[0.5372055206413685, 0.5928368013586188, 0.6573133294892068, 0.5374115029141592, 0.6602056066205864], [1.9616460106413682, 1.6156225613586188, 1.703980719489207, 1.2695247529141591, 1.0041832066205865], [1.0744130706413686, 0.4701168386413812, 1.359966639489207, 0.41461010708584084, 1.0314776866205864], [1.5385203493586315, 0.15909641864138122, 0.13717973051079316, 0.5453653570858408, 0.8814896633794136], [1.6118171993586317, 1.5673263213586188, 1.326134590510793, 1.4726586329141593, 0.8826619433794136], [0.5984461206413686, 0.16837563864138122, 0.8150825005107931, 0.9611492729141592, 0.30155969662058646], [2.4057405406413683, 0.41223256864138125, 0.5476271994892068, 1.0319666370858407, 1.0726160866205865], [0.3434082906413686, 1.5982674913586188, 1.8723901094892068, 0.27087892291415916, 0.8321460466205863], [1.8112422793586314, 0.3584416713586188, 1.368907769489207, 1.4522021829141591, 0.6802225833794135], [0.5821521993586314, 0.7353147786413812, 0.4400074605107932, 0.8411178070858408, 0.9918431333794135], [2.0361306306413685, 1.5784859813586187, 0.5910599705107932, 1.066333087085841, 0.9120692866205864], [1.7535085006413684, 0.8584089313586187, 0.013849869489206829, 1.2393140029141594, 1.1499332366205863], [1.2024002393586315, 0.7690992586413812, 1.260610709489207, 0.6121367029141592, 0.05949118337941356], [1.8199925193586313, 1.3128015413586187, 1.4869526294892068, 0.8045397470858409, 0.8744123233794135], [0.1628136406413685, 0.7600097013586189, 0.04836350051079316, 0.43216193291415916, 0.14167889662058644], [2.1160439806413684, 0.7537739986413812, 0.7087970505107932, 1.6066007429141593, 1.1102920866205863], [0.6993418706413687, 0.6818545913586188, 1.1003006605107932, 0.7898727870858409, 1.5284973666205863], [1.5287127193586314, 1.5514224913586188, 0.8912161794892068, 0.5784162170858409, 0.5358461366205864], [1.5545632193586316, 1.1414656086413812, 2.0700067794892067, 1.471883982914159, 0.5639300433794135], [0.6215065406413686, 0.5204332613586188, 1.6082019394892069, 1.1521913629141594, 0.19455191662058643], [2.2133973906413686, 0.9632116813586189, 0.26206103948920684, 1.4269814570858408, 1.6979812166205863], [3.8979347393586314, 17.881249388641383, 4.564763720510793, 20.936978237085842, 21.673184476620584], [15.538708519358632, 1.4788447013586188, 10.751114370510795, 37.21266205291416, 1.0565376866205864], [4.083906759358631, 21.153277658641382, 13.019500470510794, 9.507989307085841, 1.6890888566205864], [25.90445029064137, 6.077540041358619, 23.840131870510792, 11.09719253708584, 8.560629106620587], [35.321847960641364, 1.7803465986413811, 24.147085740510793, 14.56649073291416, 18.184048423379416], [8.909740279358632, 20.75813338135862, 8.175759110510793, 8.40048321708584, 17.028948623379414], [22.24713267064137, 1.2133489986413812, 11.052352279489206, 19.12239167291416, 0.029154176620586447], [18.61108743935863, 3.6030598313586184, 9.257341889489206, 4.696337507085841, 13.795398106620587], [30.04474074064137, 30.71502317864138, 25.818021819489207, 43.89260997708584, 11.963228306620588]]                                         [[0.5372055206413685, 0.5928368013586188, 0.6573133294892068, 0.5374115029141592, 0.6602056066205864], [1.9616460106413682, 1.6156225613586188, 1.703980719489207, 1.2695247529141591, 1.0041832066205865], [1.0744130706413686, 0.4701168386413812, 1.359966639489207, 0.41461010708584084, 1.0314776866205864], [1.5385203493586315, 0.15909641864138122, 0.13717973051079316, 0.5453653570858408, 0.8814896633794136], [1.6118171993586317, 1.5673263213586188, 1.326134590510793, 1.4726586329141593, 0.8826619433794136], [0.5984461206413686, 0.16837563864138122, 0.8150825005107931, 0.9611492729141592, 0.30155969662058646], [2.4057405406413683, 0.41223256864138125, 0.5476271994892068, 1.0319666370858407, 1.0726160866205865], [0.3434082906413686, 1.5982674913586188, 1.8723901094892068, 0.27087892291415916, 0.8321460466205863], [1.8112422793586314, 0.3584416713586188, 1.368907769489207, 1.4522021829141591, 0.6802225833794135], [0.5821521993586314, 0.7353147786413812, 0.4400074605107932, 0.8411178070858408, 0.9918431333794135], [2.0361306306413685, 1.5784859813586187, 0.5910599705107932, 1.066333087085841, 0.9120692866205864], [1.7535085006413684, 0.8584089313586187, 0.013849869489206829, 1.2393140029141594, 1.1499332366205863], [1.2024002393586315, 0.7690992586413812, 1.260610709489207, 0.6121367029141592, 0.05949118337941356], [1.8199925193586313, 1.3128015413586187, 1.4869526294892068, 0.8045397470858409, 0.8744123233794135], [0.1628136406413685, 0.7600097013586189, 0.04836350051079316, 0.43216193291415916, 0.14167889662058644], [2.1160439806413684, 0.7537739986413812, 0.7087970505107932, 1.6066007429141593, 1.1102920866205863], [0.6993418706413687, 0.6818545913586188, 1.1003006605107932, 0.7898727870858409, 1.5284973666205863], [1.5287127193586314, 1.5514224913586188, 0.8912161794892068, 0.5784162170858409, 0.5358461366205864], [1.5545632193586316, 1.1414656086413812, 2.0700067794892067, 1.471883982914159, 0.5639300433794135], [0.6215065406413686, 0.5204332613586188, 1.6082019394892069, 1.1521913629141594, 0.19455191662058643], [2.2133973906413686, 0.9632116813586189, 0.26206103948920684, 1.4269814570858408, 1.6979812166205863], [3.8979347393586314, 17.881249388641383, 4.564763720510793, 20.936978237085842, 21.673184476620584], [15.538708519358632, 1.4788447013586188, 10.751114370510795, 37.21266205291416, 1.0565376866205864], [4.083906759358631, 21.153277658641382, 13.019500470510794, 9.507989307085841, 1.6890888566205864], [25.90445029064137, 6.077540041358619, 23.840131870510792, 11.09719253708584, 8.560629106620587], [35.321847960641364, 1.7803465986413811, 24.147085740510793, 14.56649073291416, 18.184048423379416], [8.909740279358632, 20.75813338135862, 8.175759110510793, 8.40048321708584, 17.028948623379414], [22.24713267064137, 1.2133489986413812, 11.052352279489206, 19.12239167291416, 0.029154176620586447], [18.61108743935863, 3.6030598313586184, 9.257341889489206, 4.696337507085841, 13.795398106620587], [30.04474074064137, 30.71502317864138, 25.818021819489207, 43.89260997708584, 11.963228306620588]]                                         0                [190.29093558579572, 147.36880014393256, 146.50140497683964, 213.07970934497033, 131.53064352427404]  [1.1561763814550685, 2.2188499824535253, 1.984906731425127, 2.418802545572856, 1.7962344950377622, 1.963706150448905, 2.744930202959447, 2.5993043473432906, 1.731518027459286, 1.485390283978106, 2.531352127392295, 2.182782624804255, 1.7884016651084393, 2.015146372234781, 2.5135359991128063, 1.7247302540105622, 2.0928990023414533, 2.2609212667363536, 2.317379406555006, 2.50811871968372, 2.8432055120994093, 68.78344965366355, 77.21189071047112, 45.361629697326755, 57.26846147123132, 75.93853618491381, 50.29227213363671, 40.77763090229397, 42.75370569398197, 104.55790978736147]               1\n",
      "[[0.20121719935863144, 0.1139529913586188, 0.037693240510793175, 0.28254506291415915, 0.11938906662058643], [1.6624855506413683, 1.1710515713586187, 1.9919937894892068, 1.8899732829141591, 1.1822481966205864], [0.5783140906413685, 0.4008763386413812, 1.5467860394892068, 0.3070490170858408, 0.5418982266205864], [1.1504852093586315, 0.11940906135861878, 0.042538890510793154, 1.207391327085841, 0.11778813337941356], [1.6885148493586315, 1.1256468013586187, 1.340692350510793, 1.7081260029141592, 0.6282321933794136], [1.2794827706413687, 0.8166772486413812, 1.091403260510793, 0.8618616529141593, 0.8313207766205863], [1.4333163006413687, 0.08552050864138122, 1.064954409489207, 0.7907700870858408, 1.9029909666205864], [0.06395180935863143, 1.8950812413586189, 1.3271912894892068, 0.22641954291415922, 1.0009774466205865], [1.3239180393586314, 0.2647635013586188, 0.5727771994892068, 1.983682092914159, 0.6535584633794136], [0.24634430935863144, 1.1064282986413811, 0.8997980405107933, 0.46958823708584085, 0.25537342337941354], [1.6879108306413686, 1.1181827613586188, 1.3920944705107932, 1.1589210070858407, 0.6014863066205864], [1.9427112706413685, 0.28202467135861875, 0.49472512051079315, 1.3240327529141593, 1.2476818166205863], [0.6544377393586315, 0.5859495486413813, 1.632403609489207, 0.39654526291415915, 0.09460268337941356], [1.1733540193586314, 1.5134232113586188, 1.256866489489207, 1.2006293570858408, 0.6272542333794136], [0.23949734064136852, 0.7723143813586189, 0.4705247494892068, 0.29602750291415914, 0.7021286533794135], [1.5264460806413687, 0.7700023486413812, 1.2135742905107931, 1.609653922914159, 1.1911962766205864], [0.9990550106413686, 0.6845692013586188, 0.5960219705107932, 0.3228332570858408, 0.8995400866205865], [1.2592758493586316, 0.5706687313586188, 0.9390086194892069, 0.8348133470858409, 0.05148572662058645], [0.8955198593586314, 0.7014511786413813, 1.245180479489207, 1.2411333529141593, 1.2225877433794137], [1.1122484906413685, 0.7844325213586187, 1.3212535394892069, 0.5831438629141592, 0.5527831533794135], [2.1007100806413685, 1.2796036613586188, 0.022051749489206823, 0.9633767870858408, 1.5348029666205865], [0.10475638935863146, 0.3205648386413812, 1.3008583105107931, 0.47796267291415917, 1.1850343566205863], [1.5275545793586316, 0.13725060135861877, 0.04324717948920684, 1.1596413529141594, 0.13885934662058644], [0.12247942935863146, 1.6804808313586188, 1.015868119489207, 0.3390199570858408, 1.2739254633794135], [1.2897356806413685, 0.3300771786413812, 1.418567919489207, 0.3478135970858408, 0.15688496662058646], [1.5965062606413687, 0.1337059286413812, 0.9630310894892069, 1.6187502829141591, 1.6806371666205864], [0.7262578893586314, 1.4822774113586188, 0.7678305405107932, 0.20492552291415922, 0.5317817766205865], [1.1223874893586314, 0.41367247864138124, 1.2656046505107932, 0.7235409470858408, 0.18286580337941355], [0.18212948064136858, 0.7755587286413812, 0.12563675051079315, 0.43184272291415915, 0.28343105337941354], [2.1517130206413686, 1.9081560913586186, 1.0393041094892068, 1.757448992914159, 0.14619118662058642]]  [[0.20121719935863144, 0.1139529913586188, 0.037693240510793175, 0.28254506291415915, 0.11938906662058643], [1.6624855506413683, 1.1710515713586187, 1.9919937894892068, 1.8899732829141591, 1.1822481966205864], [0.5783140906413685, 0.4008763386413812, 1.5467860394892068, 0.3070490170858408, 0.5418982266205864], [1.1504852093586315, 0.11940906135861878, 0.042538890510793154, 1.207391327085841, 0.11778813337941356], [1.6885148493586315, 1.1256468013586187, 1.340692350510793, 1.7081260029141592, 0.6282321933794136], [1.2794827706413687, 0.8166772486413812, 1.091403260510793, 0.8618616529141593, 0.8313207766205863], [1.4333163006413687, 0.08552050864138122, 1.064954409489207, 0.7907700870858408, 1.9029909666205864], [0.06395180935863143, 1.8950812413586189, 1.3271912894892068, 0.22641954291415922, 1.0009774466205865], [1.3239180393586314, 0.2647635013586188, 0.5727771994892068, 1.983682092914159, 0.6535584633794136], [0.24634430935863144, 1.1064282986413811, 0.8997980405107933, 0.46958823708584085, 0.25537342337941354], [1.6879108306413686, 1.1181827613586188, 1.3920944705107932, 1.1589210070858407, 0.6014863066205864], [1.9427112706413685, 0.28202467135861875, 0.49472512051079315, 1.3240327529141593, 1.2476818166205863], [0.6544377393586315, 0.5859495486413813, 1.632403609489207, 0.39654526291415915, 0.09460268337941356], [1.1733540193586314, 1.5134232113586188, 1.256866489489207, 1.2006293570858408, 0.6272542333794136], [0.23949734064136852, 0.7723143813586189, 0.4705247494892068, 0.29602750291415914, 0.7021286533794135], [1.5264460806413687, 0.7700023486413812, 1.2135742905107931, 1.609653922914159, 1.1911962766205864], [0.9990550106413686, 0.6845692013586188, 0.5960219705107932, 0.3228332570858408, 0.8995400866205865], [1.2592758493586316, 0.5706687313586188, 0.9390086194892069, 0.8348133470858409, 0.05148572662058645], [0.8955198593586314, 0.7014511786413813, 1.245180479489207, 1.2411333529141593, 1.2225877433794137], [1.1122484906413685, 0.7844325213586187, 1.3212535394892069, 0.5831438629141592, 0.5527831533794135], [2.1007100806413685, 1.2796036613586188, 0.022051749489206823, 0.9633767870858408, 1.5348029666205865], [0.10475638935863146, 0.3205648386413812, 1.3008583105107931, 0.47796267291415917, 1.1850343566205863], [1.5275545793586316, 0.13725060135861877, 0.04324717948920684, 1.1596413529141594, 0.13885934662058644], [0.12247942935863146, 1.6804808313586188, 1.015868119489207, 0.3390199570858408, 1.2739254633794135], [1.2897356806413685, 0.3300771786413812, 1.418567919489207, 0.3478135970858408, 0.15688496662058646], [1.5965062606413687, 0.1337059286413812, 0.9630310894892069, 1.6187502829141591, 1.6806371666205864], [0.7262578893586314, 1.4822774113586188, 0.7678305405107932, 0.20492552291415922, 0.5317817766205865], [1.1223874893586314, 0.41367247864138124, 1.2656046505107932, 0.7235409470858408, 0.18286580337941355], [0.18212948064136858, 0.7755587286413812, 0.12563675051079315, 0.43184272291415915, 0.28343105337941354], [2.1517130206413686, 1.9081560913586186, 1.0393041094892068, 1.757448992914159, 0.14619118662058642]]  0                [4.901083249336203, 5.813891537441186, 5.956354489513636, 5.466678021727472, 6.185259173763845]       [2.6125559855999736, 2.7460534705995125, 2.1690036635051886, 2.444161481805575, 1.7481697714354736, 0.5294445942224737, 3.2840129708349233, 2.8154119354554488, 2.740537473043989, 1.855938181940736, 1.283551924104272, 2.0091585090291417, 2.266507886802258, 1.5999294476598176, 1.6704874095232876, 1.6441100363601082, 1.5333694856468585, 1.5934205644205262, 1.8066211792436127, 1.203220473076251, 3.291186897871839, 2.71735860104401, 2.4187485085504115, 2.6150921647603362, 2.514826815820941, 2.801157276983161, 2.0683426621080807, 1.6645318058409562, 2.2688597467479186, 3.4117162618467303]       0\n",
      "[[0.17034977064136858, 0.5472374323586188, 0.15004997051079316, 0.2757009194858408, 0.5346363466205865], [1.9529040006413685, 1.7436165113586188, 1.575935249489207, 2.028041002914159, 1.1635725566205863], [1.1713234206413685, 0.9675041356413812, 1.148188269489207, 0.7564283400858408, 1.2324180066205863], [1.6422558793586313, 0.13503135064138122, 0.3802922394892068, 0.7195026650858408, 0.30193812337941356], [1.1705125893586315, 1.3991710413586187, 0.7226523805107932, 1.0833465329141592, 0.6425567033794135], [0.9202064806413685, 0.09491186264138121, 0.7311927705107932, 0.40237524391415913, 0.14561660662058643], [1.4456380006413685, 0.2752677327413812, 0.6715783894892069, 1.5308663370858406, 1.4845644666205864], [0.22646006935863144, 1.5639882913586187, 2.035851209489207, 0.15630409608584084, 0.3851038766205864], [1.3134871393586316, 0.4213598623586188, 0.8974696494892069, 1.3354543229141593, 0.3953399433794136], [0.13076894935863145, 0.7111582606413812, 0.6719727705107932, 0.9096666120858409, 0.4042667833794136], [1.8733293306413685, 1.3015272213586189, 1.5515784205107932, 1.274195151085841, 1.2662037666205863], [1.3296561206413684, 0.13960407635861877, 0.024975779489206845, 1.3328486629141594, 1.3307280166205864], [0.9730217393586315, 0.7229526736413812, 0.942603609489207, 1.1848807929141594, 0.5300451866205864], [1.2314279193586315, 1.1170115413586188, 1.981461319489207, 1.2078783270858409, 0.4076184033794136], [0.5444659606413685, 0.2874094723586188, 0.002797890510793155, 0.7433587129141591, 0.34102001337941357], [1.5265240106413684, 0.6744355626413812, 1.126683250510793, 1.8047100029141592, 1.1081296366205864], [1.2653162206413686, 0.6699952383586187, 1.265800750510793, 0.14192792308584082, 1.1415198566205864], [1.5772017593586316, 0.9518152613586188, 0.3004966194892068, 1.185841310085841, 0.26379703337941357], [1.2945667593586314, 0.9523877236413812, 2.086913319489207, 1.0310494229141591, 0.8929002033794136], [0.9323520206413686, 0.28889317035861883, 1.199778569489207, 0.4100103059141592, 0.22514248662058642], [2.2508835706413683, 1.4367943013586189, 0.13525934948920681, 1.547246267085841, 1.5907053366205863], [4.007823960641369, 5.035389791358619, 8.521165239489205, 7.149964557085841, 26.475573703379414], [24.59185002064137, 3.225123708641381, 1.244534019489207, 28.44524729291416, 2.7237955933794136], [1.6017800693586315, 16.03329747864138, 19.45473264948921, 2.413257732914159, 17.946450483379415], [16.99575211064137, 0.2338038747413812, 15.096019440510794, 11.54200600708584, 2.7149422366205864], [20.91265848935863, 5.614921321358619, 7.287167909489207, 25.83347400708584, 24.119755876620584], [4.949291909358631, 23.27805112135862, 16.62922932051079, 15.36604560708584, 10.530424113379413], [24.29922615935863, 1.6400280113586188, 8.255588720510794, 9.82675273708584, 6.671112976620586], [17.76793598064137, 6.816176041358618, 10.311118109489206, 11.05393940708584, 3.4842001066205865], [21.99583167935863, 20.525597921358617, 22.987826070510792, 28.461865107085842, 18.678605926620584]]                                                   [[0.17034977064136858, 0.5472374323586188, 0.15004997051079316, 0.2757009194858408, 0.5346363466205865], [1.9529040006413685, 1.7436165113586188, 1.575935249489207, 2.028041002914159, 1.1635725566205863], [1.1713234206413685, 0.9675041356413812, 1.148188269489207, 0.7564283400858408, 1.2324180066205863], [1.6422558793586313, 0.13503135064138122, 0.3802922394892068, 0.7195026650858408, 0.30193812337941356], [1.1705125893586315, 1.3991710413586187, 0.7226523805107932, 1.0833465329141592, 0.6425567033794135], [0.9202064806413685, 0.09491186264138121, 0.7311927705107932, 0.40237524391415913, 0.14561660662058643], [1.4456380006413685, 0.2752677327413812, 0.6715783894892069, 1.5308663370858406, 1.4845644666205864], [0.22646006935863144, 1.5639882913586187, 2.035851209489207, 0.15630409608584084, 0.3851038766205864], [1.3134871393586316, 0.4213598623586188, 0.8974696494892069, 1.3354543229141593, 0.3953399433794136], [0.13076894935863145, 0.7111582606413812, 0.6719727705107932, 0.9096666120858409, 0.4042667833794136], [1.8733293306413685, 1.3015272213586189, 1.5515784205107932, 1.274195151085841, 1.2662037666205863], [1.3296561206413684, 0.13960407635861877, 0.024975779489206845, 1.3328486629141594, 1.3307280166205864], [0.9730217393586315, 0.7229526736413812, 0.942603609489207, 1.1848807929141594, 0.5300451866205864], [1.2314279193586315, 1.1170115413586188, 1.981461319489207, 1.2078783270858409, 0.4076184033794136], [0.5444659606413685, 0.2874094723586188, 0.002797890510793155, 0.7433587129141591, 0.34102001337941357], [1.5265240106413684, 0.6744355626413812, 1.126683250510793, 1.8047100029141592, 1.1081296366205864], [1.2653162206413686, 0.6699952383586187, 1.265800750510793, 0.14192792308584082, 1.1415198566205864], [1.5772017593586316, 0.9518152613586188, 0.3004966194892068, 1.185841310085841, 0.26379703337941357], [1.2945667593586314, 0.9523877236413812, 2.086913319489207, 1.0310494229141591, 0.8929002033794136], [0.9323520206413686, 0.28889317035861883, 1.199778569489207, 0.4100103059141592, 0.22514248662058642], [2.2508835706413683, 1.4367943013586189, 0.13525934948920681, 1.547246267085841, 1.5907053366205863], [4.007823960641369, 5.035389791358619, 8.521165239489205, 7.149964557085841, 26.475573703379414], [24.59185002064137, 3.225123708641381, 1.244534019489207, 28.44524729291416, 2.7237955933794136], [1.6017800693586315, 16.03329747864138, 19.45473264948921, 2.413257732914159, 17.946450483379415], [16.99575211064137, 0.2338038747413812, 15.096019440510794, 11.54200600708584, 2.7149422366205864], [20.91265848935863, 5.614921321358619, 7.287167909489207, 25.83347400708584, 24.119755876620584], [4.949291909358631, 23.27805112135862, 16.62922932051079, 15.36604560708584, 10.530424113379413], [24.29922615935863, 1.6400280113586188, 8.255588720510794, 9.82675273708584, 6.671112976620586], [17.76793598064137, 6.816176041358618, 10.311118109489206, 11.05393940708584, 3.4842001066205865], [21.99583167935863, 20.525597921358617, 22.987826070510792, 28.461865107085842, 18.678605926620584]]                                                   0                [156.07101106397985, 111.50319113429121, 126.72899841059991, 168.14438396455475, 158.67767028261176]  [2.079918839349741, 2.8379026106912737, 1.2403486841460185, 2.268977960409118, 1.482164182755436, 2.0667061659773625, 2.2573149692693146, 2.956401396998428, 1.382285639405637, 2.287720456899451, 1.8926963899925615, 2.4276811477345213, 1.0982376837053094, 2.1500350688920857, 2.1732317122254754, 1.9536818833464884, 2.5623174925132464, 2.016820041630398, 2.198547460356535, 1.9460702241504595, 3.215348828720806, 57.51899635431552, 53.386654429396415, 53.39495687900797, 35.58986173399972, 64.986423003486, 58.290829156478054, 40.39688674277344, 31.293342527751047, 75.91330149927131]             1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: The `gcloud ml-engine` commands have been renamed and will soon be removed. Please use `gcloud ai-platform` instead.\n",
      "WARNING: 2019-07-12 20:42:52.716976: I tensorflow/core/platform/cpu_feature_guard.cc:145] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance critical operations:  AVX2 FMA\n",
      "To enable them in non-MKL-DNN operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2019-07-12 20:42:52.728880: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:42:52.731493: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55645ed99bc0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:42:52.731529: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "2019-07-12 20:42:52.732282: I tensorflow/core/common_runtime/process_util.cc:115] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0712 20:42:52.732616 139674229736896 deprecation.py:323] From /usr/lib/google-cloud-sdk/lib/third_party/ml_sdk/cloud/ml/prediction/frameworks/tf_prediction_lib.py:210: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "W0712 20:42:54.173778 139674229736896 deprecation.py:323] From /usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "2019-07-12 20:42:54.467420: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "model_dir=$(ls ${PWD}/trained_model/lstm_unlabeled/export/exporter | tail -1)\n",
    "gcloud ml-engine local predict \\\n",
    "  --model-dir=${PWD}/trained_model/lstm_unlabeled/export/exporter/${model_dir} \\\n",
    "  --json-instances=./test_sequences.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_FEAT_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                X_TIME_ABS_RECON_ERR                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            FEAT_ANOM_FLAGS  MAHALANOBIS_DIST_FEAT                                                                                MAHALANOBIS_DIST_TIME                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             TIME_ANOM_FLAGS\n",
      "[[0.16373820183734544, 0.3542266117383235, 0.4355817535193831, 0.054669573298224354, 0.40955906406765796], [0.1293657542684008, 0.1827617576725824, 0.2715684470574288, 0.08644414535777921, 0.07438152432553503], [0.5437437228620384, 0.8101394239937798, 1.7407919810760912, 0.5615498923394175, 0.012902838293615049], [0.012025497190085332, 0.2961070050216321, 0.6646215546365886, 0.10734823479201791, 0.3897156565278287], [0.06334951329607308, 1.4158652709938169, 1.3603194672386654, 1.3215562996633459, 0.2986053033642767], [0.39660328451634563, 0.2602381890074777, 0.40495991048983543, 0.8479539872452044, 0.43186976958173423], [0.1149081374289902, 0.02255911175507852, 1.4699305215386893, 0.8674259428047445, 0.08326371004946731], [0.1883424210562864, 0.5560622805232438, 0.09164427164111122, 1.2602427872323065, 0.38055302787300715], [0.33203673303475356, 0.38597762212079473, 0.29044251568920476, 0.3079666709267139, 0.11874520807249467], [0.10727963359657397, 0.3449275850222333, 0.15852272902316936, 0.13981343879400662, 0.10973085813959742], [0.26112464731271423, 1.5501890805782361, 0.46531961500585517, 0.6570472178278879, 0.1959982535510434], [0.5353864463618294, 0.07662395061473615, 0.9119952905868536, 0.6423770766788083, 0.494728367265611], [0.45670065681000904, 1.0954033677445776, 0.8447161279555756, 0.7244545789803754, 0.3600072872424451], [0.2429524926977915, 0.8800485101041939, 0.9388256738939763, 1.591793279995032, 0.007617385922773945], [0.5699718311676619, 0.7375525980772358, 0.21610770958794312, 0.31122579889169355, 0.6985028309295116], [0.17807181941431338, 1.3047460311357393, 1.077831654283884, 2.2997644338013483, 0.6617024820192953], [0.24319409961394522, 1.030488763902994, 0.3345693440796361, 0.01965665864120847, 0.5239383822307864], [0.6873592974864838, 1.1732300033758767, 0.37758952890503145, 0.883957653428454, 0.6502498658911844], [0.24507007754170607, 1.7286171745800005, 0.4639869405345184, 0.8182365510033611, 0.5762363837569536], [0.30548780675151854, 0.10123195825980259, 0.5051485682052667, 0.3234068745720049, 1.1066996862936018], [0.3388393784381547, 0.9902173097412601, 0.429683074733757, 1.3736484288540551, 0.0912677566678548], [0.9645500624254321, 0.022919643963252967, 0.8745825431042035, 1.030304022261462, 1.2770260797378445], [0.058295485787331236, 0.8235834714179104, 1.128336298093327, 1.605795834328053, 0.16912175026880738], [0.30289540605838405, 1.3626173250661862, 0.8035632235349046, 1.5295284033196963, 0.08221771297622821], [0.6497935943397899, 0.13400749209109464, 1.9961625418004814, 1.0071004396702894, 0.6794654074626061], [0.23899489597363488, 1.2226972251500867, 0.14304457573015883, 0.615906852087191, 0.510995866073676], [0.7479399142525872, 0.8064950723331707, 1.6175169726085112, 0.04990553071041354, 0.8328114528864548], [0.04983549883811236, 0.43498822068734283, 0.16454216987535905, 0.8124693364347801, 0.28902166067414903], [0.2862940057119884, 0.646089169128788, 0.203405764127073, 0.3692911993789811, 0.3545750459341984], [0.6592731719655704, 0.41543339305160265, 0.10034180920109757, 0.18524943330068758, 1.549389352893575]]  [[0.1729249045107869, 0.3801611767222136, 0.4030211410145257, 0.014692949818311107, 0.37411241882082724], [0.47149008331733566, 1.3482907447888262, 0.7862226970452417, 0.8846580694714556, 0.5997259955542591], [0.07635155912033742, 0.9317281081546562, 1.4758490221150717, 0.8434482950317854, 0.0907488549566654], [0.036283919471545634, 0.2826233032664571, 0.2094228120133142, 0.5477506353058643, 0.03544580831460831], [0.04745131357127663, 1.6307625241330355, 0.9164171137289052, 1.7216600164741915, 0.43710181609729215], [0.08224621381750141, 0.5942944942747547, 0.885413176927021, 0.4171745317684355, 0.12935924381752884], [0.0964959936790486, 0.44972556328162006, 0.6664453412739542, 1.6252070125225728, 0.1050201214925055], [0.27277542729234655, 1.393536073303297, 1.0190237648795992, 0.42532372269945684, 0.4593189817608837], [0.13362207481414945, 0.1720157353688644, 1.1977327263368374, 1.1532837248608163, 0.12219209012764431], [0.2483346562021015, 1.1879535442558253, 0.9863801194330131, 0.64307231204571, 0.19346984605600143], [0.13653345643483894, 1.017747664493183, 1.277692712630365, 1.401046768275437, 0.12539906033694526], [0.4903689094798138, 0.19685707142326084, 0.8274814700331674, 0.7207501932780553, 0.9014014622354681], [0.355133467954196, 0.886288563056266, 1.1885365782280435, 1.0400003989004147, 0.39065418520306355], [0.15156610380585978, 1.087590889119385, 1.4365498878049983, 1.1472617867488986, 0.26067530208957823], [0.3139437904123866, 0.8427682677665687, 0.35624388482028363, 0.4175172983753964, 0.790558590548364], [0.18810134674980072, 1.703237165713078, 1.7810859195328204, 1.6372436286474328, 0.45439897138446883], [0.20893705810616592, 0.6026253568987798, 0.9828115163158452, 0.6316716515090086, 0.6323360563628908], [0.3257851018160691, 1.1962261295833303, 0.5815469395226487, 0.6868791855501922, 0.6778904315586924], [0.17160294593154868, 1.311886376111304, 1.1756745243109297, 1.4653492740200522, 0.7271850870535521], [0.3586579297833382, 0.31851468839204805, 0.9204186098474431, 0.061250058483455466, 0.8413075917850716], [0.10822768729717858, 0.6474526987375665, 0.23315989072359394, 2.011355140851473, 0.214756016068958], [0.6049667782925536, 0.6810125310423875, 1.6686897519312185, 0.29666513522236854, 1.3521453585704897], [0.13138861979289462, 0.6839980621663786, 0.7530913052114587, 1.9376822751043956, 0.04279647144959697], [0.12375951361359117, 1.6407469089855375, 1.2560417720759354, 1.1289109308558616, 0.21423619114443132], [0.40006379669518677, 0.32823050671140813, 1.6251444748077561, 1.375447238019315, 0.6628171571091142], [0.46115216036162443, 0.8679795369906039, 0.05078495588149021, 0.7811222558298989, 0.7813473591110848], [0.5088319009681943, 1.0683174885522713, 1.2739987713781387, 0.37476634356787314, 1.0227542283722448], [0.3084159817373453, 0.5716527806998117, 1.0901061221425516, 1.6828785860348314, 0.10508434271625655], [0.2853360089837597, 1.035701978983373, 0.19125958039486624, 0.012685528818662924, 0.49696178329169827], [0.3943216266267058, 1.4528615352130994, 0.8768599809733509, 0.7127534340726664, 1.0983471253254273]]            0                [4.492165225483232, 5.925728806034316, 5.585922775356791, 6.454044446633134, 5.94465340680206]       [2.1946417128865154, 2.191082462281185, 1.7222515497374202, 2.2663024990465326, 3.151111026475872, 1.69187184759487, 1.7753724410159235, 1.6468076901829776, 2.103685750444875, 1.7185026237179812, 1.476564934172033, 2.3717720681568304, 1.549278025137391, 1.299975004234112, 1.8043086718248087, 2.740195415762323, 1.2981770760288147, 1.491269525122406, 2.5130721221064882, 2.327722231470522, 2.7048875628811535, 3.599131070651172, 2.247458355970372, 2.0764781191091473, 2.835313229248439, 2.3458066340278787, 2.2190809368238056, 2.695950541051843, 2.402259443391408, 2.671490322815947]           0\n",
      "[[0.3239943922469582, 0.209878586824551, 0.40664393865737214, 0.06735433092095196, 0.08635235748420039], [0.318935021709441, 0.10420873096826685, 0.3853263664440988, 0.506674028626845, 0.546430586023236], [0.12333201375132785, 0.5942222417767056, 1.4982573882083345, 0.8898864642218908, 0.1104651466260208], [0.33618985032273707, 0.03466496286629031, 0.1633400086214479, 0.3047817267483343, 0.5785945478968605], [0.4308296858240488, 1.6538082391184425, 2.0810029719939593, 0.6903245400187056, 0.4556432067660431], [0.4853786757097215, 0.440072671833553, 0.059764073420138064, 0.5979445325412129, 0.19456016360165845], [0.35646054473290345, 0.6944121075606593, 1.144724618238647, 1.203643380897883, 0.30345036789819174], [0.3702930104060169, 0.26517488367281417, 0.12884060453339502, 0.958574753271244, 0.2826071190373993], [0.17317115092946578, 0.48812159426567553, 0.17799359232880918, 0.6719686304966744, 0.16755293171913044], [0.04936301991871952, 0.11606090394894951, 0.49478169097109037, 0.4672185596088044, 0.411458783099439], [0.4187383439348923, 1.6746503983861936, 0.36120901696005236, 0.9449844882957424, 0.39512759297890954], [0.20524372519863365, 0.07000204696846615, 0.743455447410788, 1.2498897470365395, 0.08406437639916983], [0.1524392977347695, 1.3052400448738037, 1.0483779471149848, 0.7833828994215871, 0.7596806489690197], [0.029643805702217252, 0.6137092946113685, 1.3761982180723495, 1.252951198879214, 0.307891405144391], [0.2570436813014101, 0.25123233911493337, 0.4166965302216967, 0.2749089659929239, 0.9954083513720388], [0.5734168373918838, 0.8321639587426036, 0.47249787845999514, 1.9340127947408572, 0.3357863325351899], [0.6928804504164614, 0.9238642789242664, 0.5010326834861565, 0.002020456170916285, 0.7297539704602745], [0.6446774601720969, 1.1790726534496865, 0.09498532580576424, 1.3853658100273378, 0.7484750142172112], [0.6772618508371636, 1.5110930874832333, 0.4984331378731274, 0.6761337128714426, 0.2286749734463152], [0.5893329485788614, 0.19276332541609031, 0.018248581367358768, 0.3861612368125638, 0.41645402704331047], [0.2771267775166051, 1.3184703658675447, 0.5201374970627908, 1.1216837631993442, 0.3083257434495513], [0.4887423887780964, 0.007149306763392843, 0.8503384706251157, 0.8274021648256082, 0.8347775855419595], [0.07235008172017432, 0.04689013516413448, 0.5119403210559547, 1.1956842101381313, 0.31112981319196953], [0.7132448251886481, 1.0191432167473522, 0.17112903259809292, 1.1333646247643032, 0.7603521073465804], [0.3662138476005775, 0.2277163662568743, 1.8571858424166816, 0.44465792138383425, 0.9610181144239679], [0.18750471873745345, 1.0950773842567254, 0.029819161234992453, 1.1860024682531665, 0.10109195399104415], [1.0018427963411098, 0.7210155659303193, 1.7391307033535164, 0.12958355502060154, 1.0938949881679132], [0.552326720808829, 0.47025537012647617, 0.47124929516298947, 0.056861140945606126, 0.9968454204273458], [0.5417171194839628, 0.8660197978113282, 0.15392693716248262, 0.8003936073531396, 0.2704849539785992], [0.9785700150287476, 0.26416189790171685, 0.5261482056094825, 0.21277721379791203, 1.2822291508633326]]     [[0.0898579249077206, 0.18907746900419642, 0.42668390514429616, 0.08293687099001337, 0.2348831307924288], [0.15903679006752558, 1.2949896406482584, 0.6922418181324209, 1.469652257939351, 0.13440047080887363], [0.022561832756327838, 0.6364786978717316, 1.3221488561478794, 1.115488446858759, 0.07842454941055704], [0.05918917364946852, 0.6838177793727064, 0.6973916669356163, 0.7951438315534216, 0.16082998357322298], [0.009189884151310945, 1.7388863678473492, 1.74693394606617, 1.1071890221017557, 0.14227291111505735], [0.08230567769270469, 0.6988079873331314, 0.4972708456627592, 0.09377187051061892, 0.09784762679378112], [0.15666056877607404, 0.9766427357248851, 0.5183933377435462, 1.9358364475311134, 0.09217207097105684], [0.41895468002085245, 1.0309973452453456, 1.0010120294190985, 0.0978204415007545, 0.6974465814655835], [0.2809914795314654, 0.15627895967259142, 0.8452565513038777, 1.397283360602586, 0.1715263566370856], [0.03406763533143964, 0.7223983456921255, 0.3306116395108444, 0.320545337292346, 0.06994971896891372], [0.29571617212349677, 1.3232553791138535, 0.9883472593278418, 1.6141206278766562, 0.19049349025451556], [0.17527679368362348, 0.29339968176998615, 0.5865576889766517, 1.3053363678597567, 0.11860196022485892], [0.5055046852795545, 1.1900047808780705, 1.2863747836316421, 1.030165950476896, 0.6943843823300335], [0.07258445726956886, 0.6491673038736969, 1.714507190613357, 0.8285588724803785, 0.18333323337899654], [0.3511903729359105, 0.3329560660209298, 0.5459659600635887, 0.4118368890426527, 0.8614335700720539], [0.02943599536575947, 1.0663244850734803, 1.0379239973914915, 1.2465875657266667, 0.04817640191571293], [0.3065206171852759, 0.63470303347165, 1.0036819478276493, 0.5413588142410277, 0.8076588729780529], [0.35655407961500907, 1.0988442331462276, 0.19693703299011542, 1.2199190790556673, 0.9004489206416884], [0.03138246005951073, 1.2900876634803435, 1.0021289199470087, 1.2186018829203107, 0.3773137882159976], [0.16142227899952977, 0.5928033116842983, 0.39832567474701996, 0.05570800761316022, 0.3790449912425677], [0.25012934428747036, 1.173921471849691, 0.9663458026321226, 1.6350406299968583, 0.09115130744165878], [0.2617965523087458, 0.5607837390420027, 1.561973504371456, 0.10109756782924796, 0.6952394937255028], [0.27696211018222283, 0.039985673611554834, 0.2823736778361204, 1.4798674207519082, 0.2609541730316788], [0.4172433638745038, 1.1815350536853553, 0.5080749578318933, 0.7717722769000045, 0.7715646698039462], [0.4641385076584241, 0.3182804574342022, 1.5978207257413224, 0.752994798464632, 0.9216117791904455], [0.22113284699093994, 0.659433723213572, 0.2483877916120569, 1.3895200960224738, 0.1423571930907791], [0.6724277252052473, 0.9332194852927295, 1.4404175523823854, 0.1930261365244123, 1.4113020965256826], [0.16623412413096794, 0.5600102627738163, 1.4153017107556993, 0.9196267292506644, 0.7089773314514326], [0.27857485132393733, 1.2555439988999202, 0.2494768528457218, 0.3914000647296313, 0.5543158092602949], [0.3000598346283774, 1.2973261703320835, 1.4853201499609392, 1.0671385760455034, 1.047302102325408]]             0                [4.07096684821713, 5.405706452197568, 6.426690408633658, 5.920318809252212, 5.306628637855666]       [2.4763427111634453, 1.9970398393798028, 1.687731427206344, 1.233897305432991, 2.787545651736269, 2.3225668736116307, 2.4748353547782957, 1.9105910231534247, 2.4150619049947566, 2.2616348584175516, 2.4069542302595908, 1.745984173343468, 2.202404262378663, 2.214069656875039, 1.9671007634636757, 1.5257683189185565, 1.5226936452758395, 2.6988213152335363, 2.1066031348212846, 2.0448172068003, 2.319605170015566, 2.574226311694161, 2.4953426437478488, 1.8034572724824354, 2.9104250443250113, 2.0391765454069475, 3.504772175062107, 2.2519858296162494, 2.1874297121127175, 2.8578795595180173]      0\n",
      "[[0.06290282137115158, 0.4980054981920246, 0.2956961473025257, 0.06374060014599635, 0.4406259451683316], [0.17638763285539558, 0.11849440447156882, 0.08141094429636872, 0.6158787763333612, 0.022170087316273568], [0.2829787554525427, 1.1708482452981228, 0.8295718328912467, 0.385147265186913, 0.2766455036426241], [0.30720173090382413, 0.2298653589156938, 0.6992187893550559, 0.8248652185734022, 0.7410684055097018], [0.06499648106148559, 1.431262054619229, 1.374550739250691, 0.6166360272121347, 0.2940452052355111], [0.05097197696763478, 0.6863614819579492, 0.7410477903762412, 0.7711532199030151, 0.5796399284779252], [0.31652150445507354, 0.960065262846636, 0.9753651473134365, 1.006435762840593, 0.3117024219788149], [0.3153271441000792, 0.30016577314806314, 0.3563069293208731, 0.7093323906312248, 0.41874001187099075], [0.3527851467757117, 0.5567256245071861, 0.25872811944946084, 1.0483376650638945, 0.3098246791560494], [0.22651277875386427, 0.2671945599970439, 0.32018648835269237, 0.14900016222539914, 0.1965483027187518], [0.368174479209469, 1.6615754395076932, 1.1550880256242666, 0.07201978809808551, 0.11640884150039676], [0.1802123330756833, 0.16454431993850321, 0.1261379325644914, 0.6837488721301899, 0.40665035501581215], [0.1559349423750399, 0.8410820094585616, 0.4179336925106029, 0.4962343253541168, 0.08858903382084982], [0.04965815302340615, 0.5674878299353817, 1.2224388209971875, 1.4048539095412744, 0.37826973576161593], [0.5348789953126577, 0.8503683121747513, 0.26668052912714457, 0.04285997779418073, 1.0011159259937752], [0.32006527946284047, 0.6168499029104668, 0.9098632037978858, 2.2420021024254195, 0.1399857891190759], [0.6666221232444565, 0.65445127969205, 0.19662829102727286, 0.006604291479390678, 0.14228581656312622], [0.18729736934912466, 1.1848193161580238, 0.4697764552104445, 1.0903720600906228, 0.4305543887640814], [0.2263822171829506, 1.6223106449472857, 0.6608135307224317, 0.613331775053223, 0.6459879551597324], [0.6606161435024043, 0.30323935097832416, 0.3756684059444196, 0.4490647481841078, 0.7802391338432154], [0.10475315156089016, 1.490189289063387, 0.3522695865981884, 1.3752997165070007, 0.45557227014306223], [0.767529420794238, 0.514681825815589, 0.171927570258046, 1.026298835631354, 1.047427903321226], [0.01586024500322125, 0.231667030957856, 1.301823948174736, 1.6549082383101292, 0.797044056802977], [0.7523492178605928, 0.6064892926886667, 0.7878815537202483, 0.6873873949338416, 0.5649683732424275], [0.6424038537362543, 0.3606136646447716, 2.1169224840943897, 0.8772265791272287, 1.170434632963486], [0.42845003396204384, 0.7714381856678354, 0.3477155250485627, 0.7827757850808135, 0.22925711849602864], [0.19445907818749747, 0.5777331919298145, 1.7741449732721855, 0.04653483757499183, 0.5790121105073499], [0.03656869464675597, 0.18183764713047176, 0.2875848069892379, 0.09211017706595495, 0.8244416799126919], [0.028350410624789414, 0.9691465625760824, 0.022560742356174803, 0.9998183320712773, 0.7273123137339453], [0.7615445188583024, 0.4330391396512474, 0.2632176416276707, 0.6152594789420424, 1.2205538083573937]]                    [[0.19547644045066143, 0.46611974701454184, 0.24835379361926574, 0.005210474090488051, 0.3414228220090527], [0.44442222577749635, 1.3015429354451455, 1.132486485279324, 1.5641664994383233, 0.36363803726768595], [0.056718062382066226, 1.2298309839624846, 0.5935567759475117, 0.63466133034975, 0.2390422942264454], [0.03740943579177958, 0.8687760104762792, 0.20701078300257103, 1.3162028560858672, 0.225129232815664], [0.11826146755798561, 1.5247005384159675, 0.9829839062241464, 0.9620612156132771, 0.0678060536347378], [0.23705035839132726, 0.9447575046227616, 1.2027994762539813, 0.34451843164185686, 0.37077183328192387], [0.26890631713499213, 1.2698823197486953, 0.22996500064058156, 1.7167338644337127, 0.13323712356911221], [0.022023474570320473, 1.0734139270783283, 1.2680857183552343, 0.12265915508186634, 0.25755544177650536], [0.06370600834598705, 0.18739301372124315, 0.5565517639480424, 1.7856548007142488, 0.5639148883082606], [0.21573861481619017, 1.1087080068758242, 0.5142692619711596, 0.6488089094708529, 0.18291264154898945], [0.11343414920462247, 1.2606114378963489, 1.9500736546385184, 0.8413399040632957, 0.11385886932892986], [0.34620788369790745, 0.17968672942566427, 0.04264827678004916, 0.7261545693988354, 0.5282490380773317], [0.14511595724606885, 0.7043036957329549, 0.7310054068338543, 0.7801204452063613, 0.07146517053105245], [0.22454191342520824, 0.6349934628075964, 1.6861327003112945, 0.9667373130483854, 0.4517806786480576], [0.4545032459912172, 0.9331782069940695, 0.40689064002839526, 0.07428307288767015, 1.005563320511242], [0.1252402358357767, 0.8759170510349892, 1.5835390036042591, 1.5766069405642953, 0.12261865784739157], [0.15669331909271111, 0.3471781565323389, 0.7783297530357005, 0.5197441583113076, 0.4933498962148803], [0.11320623381569406, 1.1160029584878284, 0.6265900183708228, 0.9509458989424456, 0.31096790493674875], [0.11757076654579746, 1.3498455529265785, 1.339364670964324, 1.2579716385136364, 0.5914658687516677], [0.24811344270702007, 0.0985671900066833, 0.7624371201298719, 0.7809401455181934, 0.7494187481926085], [0.21493463267824953, 1.312035306297366, 0.22440656173414403, 1.9061062555105355, 0.05059531108856796], [0.4994051747310671, 1.0965662031825965, 0.9424518188368516, 0.3070490926412125, 1.0788610242238625], [0.3699302483207172, 0.21900565929058707, 0.9862666158589088, 1.918900399998173, 0.4401187114147577], [0.35941904724592594, 0.7847686238081476, 1.1937456664839088, 0.3394470250772047, 0.8074201513618242], [0.5751195287520924, 0.4702004569200166, 1.7854740663390956, 1.2007982375036705, 1.0595631267758416], [0.2563826644616478, 0.36268346662612844, 0.5395274435667933, 0.9178613188074413, 0.2686956025419992], [0.27666995140119993, 0.7889327792433225, 1.4642896779291703, 0.3231017053489374, 0.6059544842580581], [0.043126648526415456, 0.8527291376132246, 1.235240690441948, 0.9967408042369672, 0.30284054846838504], [0.3379839712342319, 1.3467495667533427, 0.35307877846659236, 0.653059234994448, 0.713244552425492], [0.2006670514557749, 1.4680439831914667, 1.2255929468560998, 1.4825080258171073, 0.9183088466084701]]  0                [4.797414995724164, 6.765524480734105, 6.348788133422567, 6.230285207270649, 5.436413543094264]      [2.276423062387344, 2.85734545481326, 1.965923875566993, 2.2099501174452234, 2.112748681212313, 1.465707827060622, 3.1042354502572724, 2.477525728350522, 3.145569440556514, 1.7837783334509163, 2.5096277292554334, 2.143857429973517, 1.3991503731115058, 1.9292134112643669, 2.3979596734592192, 2.0099678487108634, 1.5679242174382875, 1.3730957623349385, 2.2395175591324916, 2.174740828524526, 3.3506581683764125, 2.2252488574760947, 2.9359065994992366, 1.653939216788122, 3.568277271724903, 1.4737835164568254, 1.8154801468911799, 1.4775264728164879, 2.157693715305271, 3.2525438252742567]       0\n",
      "[[0.0798940917295807, 0.3346276620138972, 0.2678634873581634, 0.22417462534852045, 0.24126487885458117], [0.11916143038694343, 0.00032125697141394127, 0.20152993165547142, 0.07792816037874972, 0.3247621200387074], [0.031245825677229, 1.1009933941942074, 1.3554587689245903, 0.9741743592971371, 0.535827901124724], [0.24052626311018555, 0.08846739053435834, 0.02708962213772126, 0.25883081576539063, 0.3024683080854754], [0.2128046439869986, 1.0584099109924674, 1.6902254146256919, 0.7761100824048088, 0.005403793945739421], [0.5732204997422634, 0.5963913736626938, 0.007528578975794342, 1.4337591874288982, 0.14270276662853915], [0.1912976889407001, 0.6637452013086775, 1.0146697529218802, 1.0135384576529176, 0.685402751602471], [0.09913390776717002, 0.5210488491811904, 0.5707219006150663, 0.669109144874299, 0.11457770876225637], [0.3635799093508776, 1.1841345135266335, 0.17101092031676468, 0.5301232225585155, 0.02252836332212027], [0.4879193764845976, 0.5493169237857235, 0.49861404098309936, 0.38498390575014757, 0.35549020518227725], [0.1192115936595679, 1.410591912035268, 0.8479843255165382, 0.5299502542927893, 0.597305580518028], [0.004885949807721057, 0.07415026828003846, 0.6332051071106083, 0.6635928647128182, 0.00789428126277325], [0.33994207695415124, 1.9005880232806553, 1.0603886044094981, 0.45426993884807276, 0.410434888039806], [0.15720573181025554, 0.6300568290478427, 0.5535403480828939, 1.5473710994786603, 0.265675575187372], [0.44019206387090826, 0.22418560131362014, 0.16566961115066695, 0.07056384864996487, 0.7275314777240344], [0.46604271777026396, 0.8872044895676127, 0.5206883980767332, 1.5830730088735647, 0.03850546893287199], [0.7676461919822997, 0.5960183007394728, 0.6823746442736409, 0.24324852430442157, 0.19058702270407174], [0.7923348401824272, 0.2933448912819626, 0.19243682081088143, 1.479535095162602, 0.2446675912821349], [0.4403670714142036, 1.3781716222584746, 0.9202568900914331, 0.7686006557039236, 0.07519513409035727], [0.5869460862719287, 0.20250519884001628, 0.5546288257802343, 0.47724382490005984, 1.2630040384987662], [0.11290724552020981, 0.9706167816535535, 0.2515041669999605, 1.0597735798980101, 0.05436571629420328], [0.07920534658291234, 0.522730581152498, 0.8131101589167719, 0.4279825158878432, 0.9353481654635021], [0.3384925327225745, 0.10828121052796708, 0.6592399953603809, 1.4163242182117761, 0.5724090702544454], [0.040763451383934, 0.6959264874220286, 0.11402146124356694, 1.3092699754602592, 1.1169905756242553], [0.5833564937066583, 1.0767377738757082, 2.130634237893843, 1.2431051832163065, 0.9177887669312268], [0.005026425149501468, 1.2426983076819929, 0.09014072809302803, 0.9734695127992464, 0.37366767665483874], [0.6511703388946419, 1.2519265215523572, 1.0107384869018916, 0.4005790516684665, 1.1389625255036395], [0.3524469205344797, 0.5205765706328075, 0.61152075123666, 0.6590638286654453, 0.6470688781666565], [0.7658559238477871, 0.8355368805784272, 0.20084658994575694, 0.9698299955286943, 0.8385695037504904], [0.5686610763624239, 0.06276937440317734, 0.30836642153570337, 0.23832263347188584, 1.196583353399086]]      [[0.15619407373428656, 0.3067494415379041, 0.2305647483048995, 0.17492400408462774, 0.23365230893177044], [0.3361874211928566, 1.160573098414901, 1.2633095644178494, 1.0587357685504581, 0.24931190119994096], [0.17993560064412817, 1.2357339489193737, 1.1007829287867656, 1.2312080373044325, 0.5568342935394259], [0.1603689869500573, 0.6520499329976418, 0.4828563250762118, 0.7045690765295682, 0.08473515105032037], [0.0008005229337519904, 1.3088925541004441, 1.2427463663071663, 1.1738442648957013, 0.173001085527279], [0.018543048106156812, 0.9808750448138313, 0.5399735897562692, 0.8780395758290345, 0.12272432667534783], [0.1859413622063968, 1.1909762899986294, 0.19238039443628355, 1.7951531788062414, 0.7880828140925611], [0.18386280637290237, 1.385654227764584, 1.5163441056137172, 0.19612043938593604, 0.09449718192311654], [0.08866836234908804, 0.5803303654659834, 1.064215340295187, 1.3337015192787907, 0.20207871876366557], [0.18673463342464935, 1.4039604097800225, 0.35432128690909215, 0.4466219090315964, 0.17321236330385348], [0.37245053828996866, 0.8305581699431386, 1.6731810064933241, 1.2874235056913457, 0.3160104895050167], [0.17073610002765216, 0.16615774204687858, 0.5706698321531986, 0.7055816004542019, 0.2426608083512487], [0.34093103131905245, 1.6698700744722186, 1.4072045179306858, 0.7700831863698703, 0.4315152576280623], [0.003519120108870233, 0.8866223937323705, 1.0661028219319806, 1.0855885470458182, 0.06993754784045769], [0.33737390474707446, 0.34180923822836734, 0.013805367261253247, 0.19772442548391292, 0.7267220330191556], [0.040098270412306736, 1.3463985528156424, 1.2665251429370854, 0.8490790293255519, 0.06310603264447157], [0.1584649960827848, 0.14370269346815678, 1.312358995785409, 0.3242576549836084, 0.5225838510874429], [0.11818556229170385, 0.3523966721013306, 0.039343070860568374, 1.2347671358367258, 0.4806583661698746], [0.10768356720930505, 0.9198516918861692, 1.624635232050201, 1.389161350773048, 0.22268755187316824], [0.40235651291176244, 0.6194477000382103, 0.9576644120520275, 0.8373458701098642, 1.1127952180066112], [0.11824684226078475, 0.5813780762915293, 0.9200680923094273, 1.6965588495081978, 0.2055408837567978], [0.20381333943209473, 1.2183276549024342, 1.6436620400481754, 0.3707957597016502, 0.7199894104860491], [0.39930792408701743, 0.07260041229989048, 0.2684402160798151, 1.7690670530357024, 0.4388052552685574], [0.4695772824997293, 1.021882205806982, 0.3883302964970424, 0.8185512034685254, 0.8451861361186284], [0.5326849321074381, 1.293876596799653, 1.7606976835259087, 1.6029196583332466, 0.8491331642793906], [0.3805849413608182, 0.9133457616210982, 0.26753441613297857, 1.1116294316803692, 0.5282795067402553], [0.6855049159056993, 1.522362198242405, 0.6769979006261116, 0.7028348888484844, 1.2172534057015234], [0.08024223759639226, 0.47457930254970937, 1.5247052790853188, 1.5070022611885645, 0.3538962546532981], [0.5323560643124723, 1.232087404165052, 0.20730601428746578, 0.5618403524022341, 1.1009889896617369], [0.22924495412900092, 1.097084923635811, 1.280926211803542, 0.6509262539534617, 0.7855448791484997]]     0                [5.025810323875932, 5.899262244749986, 4.678558125708746, 5.504169464986474, 5.145063449933794]      [2.2623885106535537, 2.0382633514571977, 1.5383285300694431, 1.6181622757493592, 1.9447547383624701, 1.739427620581029, 3.686475472760985, 2.8010475121387666, 1.2776853894978437, 2.569036204587513, 2.6340428055772636, 1.6630856206231344, 2.2617311792592774, 1.4670213461289137, 2.349608849771853, 1.9171722823536834, 2.6161292029753653, 2.380751996849664, 1.804531343213955, 2.42040711034105, 1.6670976702897626, 2.364227778206759, 3.107899588199854, 2.0210413652757313, 3.1806918771240285, 1.9157858248936006, 3.397078409970654, 2.2911558802248337, 2.9205350520680793, 1.8545897360177395]     0\n",
      "[[0.22460295011193, 0.2136917468590264, 0.4082209164403445, 0.4532689275930162, 0.09255318378666696], [0.4260313677583272, 0.35726595759797747, 0.5126124747452279, 0.006650166051629852, 0.3781686261830144], [0.1059464963240605, 1.1075668453781462, 1.5038574500946467, 0.9250665358022203, 0.42163905343412733], [0.0311293155484042, 0.3487638418986544, 0.2274432436941829, 0.4985224749459569, 0.6337333588378946], [0.33244289828262685, 0.915091678238058, 1.7717912088306358, 0.9386368157067274, 0.38567610699081845], [0.5626417874172539, 0.38860829665029306, 0.09384376089707203, 0.8281994372943239, 0.018618634027366243], [0.5050159482952814, 0.7593929338900759, 1.1147726084628318, 1.2688448664427656, 0.3648375520536957], [0.11343861031647784, 0.14641943590768342, 0.05995456102920188, 0.8902888601878979, 0.30120701430543095], [0.4742976262329741, 0.5253257091852999, 0.5190609531407417, 1.1283986918966402, 0.38696121275601136], [0.43922655970952407, 0.08059042543269179, 0.1592986862050824, 0.11537782237586831, 0.3148170511862938], [0.011036651030254196, 1.3509070357227517, 0.6776777500610012, 0.45887222368131775, 0.18725525795777975], [0.27595411162190575, 0.44412368879980063, 0.49524883909578193, 1.1882998634255952, 0.4744058845645966], [0.1039740614956789, 1.5024902357143213, 1.1770814051469458, 0.47726558617911896, 0.042117346731526284], [0.29253832202462826, 0.8487285869396144, 1.3812918558919085, 1.8529330958438925, 0.1567092382321491], [0.365833041907467, 0.8634035760509802, 0.23603991398695243, 0.3638525789619438, 0.6551942110187929], [0.48875846739059137, 0.6672815510706918, 1.1092710051399952, 2.0262528673923503, 0.10774916343640772], [0.5684922330255967, 0.5522965449714332, 0.2012383429496073, 0.7664914344482894, 0.7111516590675053], [0.8614060460449092, 0.49385419027716304, 0.42408472657833585, 0.9784263341995946, 0.3322671654611669], [0.39561528697757065, 1.604825183892916, 0.5726119932249775, 0.49012072850377486, 0.2074731596234095], [0.6477248762697592, 0.3016987173692711, 0.5895658364125067, 0.47931057071744215, 0.5608297770101207], [0.03042661742013486, 1.3571671343806702, 0.17417581713235875, 0.890509586560657, 0.3773339828872443], [0.6387144456087992, 0.06641508930052009, 0.6377237327325656, 1.1080310390139494, 0.6878237433244478], [0.2985096429637706, 0.5410069379237926, 1.0744381236048686, 0.9953149365026959, 0.41841026251053703], [0.7772895355163036, 0.7905130521398431, 0.8401991228224551, 1.260368826795336, 0.4543754546322644], [0.45665630565490356, 1.0511436753549672, 1.4533537285525322, 1.3736837846816334, 0.7335644673830769], [0.14736289931401603, 1.2861304564118083, 0.2876709089658139, 0.9363049976879109, 0.04690151623218708], [0.2520304907537854, 0.7685948078679622, 0.8750376389205423, 0.12389740999428547, 1.3176770077560234], [0.031209390632205375, 0.5027278827187767, 0.47558402221684015, 0.5085965349916592, 0.1236463004533459], [0.1527733862073654, 0.508946991706325, 0.21763418573941026, 0.8011154969692933, 1.128942082124179], [0.48408159316344257, 0.15916367442914972, 0.4797673805355731, 0.5599892300734215, 0.6934286061292192]]     [[0.024702326303040756, 0.24430667835117273, 0.36411369815112665, 0.3907069658515276, 0.16233735989082287], [0.20739156963061123, 0.8092173514398959, 1.5627468245695137, 0.9596362577279816, 0.0025066328442524544], [0.135747457442563, 1.2052375097494237, 1.272654195110145, 1.1153245055112422, 0.42381522553282447], [0.023492506050788897, 0.2479526862200595, 0.7046030492583112, 1.0046280690251326, 0.25819225927878153], [0.06546297395767642, 1.1021634715418906, 1.3468631046718906, 1.2512109474189295, 0.300649356194871], [0.10652870547177873, 0.718462824446272, 0.5975883344002604, 0.38602294276107624, 0.19700299725886294], [0.3437111688265746, 1.1834310437834366, 0.3614132396926606, 1.8586476253907156, 0.2696385310160545], [0.3184517173435195, 0.9682001323952598, 0.8574050922326923, 0.10409387212702799, 0.5176834529242827], [0.12285705261701607, 0.035751335804987384, 0.29458583963074925, 1.7142744202455893, 0.6440541566810307], [0.29816768974290064, 0.908655928164634, 0.9524065143060059, 0.5973466009931621, 0.3185514236139124], [0.023707127593054267, 0.8564093252080491, 1.4581401718128093, 1.0778327081306334, 0.3070466743406882], [0.4911725981751487, 0.1501217841984307, 0.3953945353770036, 1.3324926242925796, 0.7392646653449112], [0.1358996974396527, 1.3185521597993974, 1.4885069353473592, 0.6992847104008069, 0.01470934160771803], [0.11017538343296329, 1.0074570912863705, 1.821239168894447, 1.585000448050788, 0.12288778604388362], [0.3098428876677768, 0.9648025556340843, 0.3778689083236756, 0.26552492144039197, 0.6431605591537606], [0.03772809702234481, 1.0289261809398034, 1.7847943215623425, 1.4820904686416079, 0.1518994235994633], [0.29283780680294624, 0.16452794651137673, 0.7970269112728618, 1.2287492762722654, 0.9267664916691647], [0.2127993701063422, 0.4963789522645847, 0.613560845691217, 0.8462349812697312, 0.515434784681694], [0.08837351679474614, 1.2418202306423987, 1.2167022072223572, 0.9390937100454698, 0.4561234267547746], [0.14635496623846378, 0.10322424916229272, 0.9714936883997234, 0.7954730591167649, 0.5735383160154107], [0.1845131375111031, 1.0724443657582794, 0.7582050920165415, 1.3087143996659463, 0.04938505393506598], [0.32594006222765637, 0.568040836364954, 1.4065778600804693, 0.4573425398345168, 0.7292273434591781], [0.22842624308275994, 0.43624937950984993, 0.7341775829734435, 1.2070717443858943, 0.2942007588950847], [0.3636214539931772, 1.0333875713760983, 1.2458878660019481, 0.9920600719127943, 0.6920841744526284], [0.44659038881728863, 1.2145733370832665, 1.1242476532147943, 1.625484073490043, 0.6067600724375163], [0.2696969046362132, 0.908840847209312, 0.5099484085222841, 1.18481049385695, 0.265321151163475], [0.631375886537096, 1.0028228646668127, 0.5811797582007088, 0.3329303636071478, 1.2020186417841972], [0.3447347131578944, 0.5082967242108906, 1.402573033274107, 1.3913232231889725, 0.2085073637192446], [0.5429808797388467, 0.8881799488318759, 0.5888667391471549, 0.4671808448467322, 1.0751286550772579], [0.06703112969338076, 1.1914927525696577, 1.4426397369455044, 1.4317395917639193, 0.38694506340881063]]                   0                [4.815457775697667, 5.207017111332164, 5.615621104870112, 5.391808117902241, 4.831599981708739]      [2.275313653764342, 2.485171727520916, 1.3494768879063217, 1.8544367616813409, 1.5741746473707914, 1.589635746130931, 3.072393044547609, 1.6930293844543034, 3.276189082295012, 1.3650315120980216, 1.8769014309939576, 2.7272939859987284, 2.28894310913597, 2.3453746522422017, 1.7454569310769883, 2.2738362427685854, 2.7195169910876578, 0.9599771488548005, 1.727642907674408, 2.2008257818035224, 1.8163312603795585, 1.9256916248843163, 1.1990684538026823, 1.2391486554374231, 2.4190334517443217, 1.5229751636518414, 2.8226638610798593, 2.7156512677197733, 2.2802047781698165, 2.0220758539383317]  0\n",
      "[[0.17532604344903835, 0.38786752518852485, 0.3149394224662538, 0.026627971596989302, 0.19591786831588762], [0.035364831854575424, 0.35916422249102986, 0.08506878617686708, 0.017606707909608194, 0.750179221380275], [0.5281137536943432, 1.4030499680995443, 1.370431696596433, 0.23901668558304046, 0.5262316331678243], [0.5492333087339333, 0.08830855709151891, 0.2082151410158713, 0.8463019669786394, 0.19189077866932383], [0.32387466648546237, 1.365467436803679, 1.8865780196365185, 0.863208747980303, 0.37585271941149007], [0.2833016114295793, 0.3229325316750929, 0.3271202103053136, 1.3357574910316308, 0.48597667270474104], [0.46533864805764824, 0.3997927083125075, 0.901182266382942, 0.6461588854433995, 0.4956056751224883], [0.39315605048865787, 0.2749430752644566, 0.678174879966108, 0.9818955074801126, 0.11488646710252004], [0.23201422283773265, 0.9245675468137726, 0.2980752040616854, 0.6698625002869941, 0.11958570942499713], [0.21706114091463422, 0.051603469040781315, 0.03949138717459283, 0.17889666576478586, 0.013694559330865538], [0.04592883090856614, 1.471282033497599, 0.4982591213478278, 0.9095864445091298, 0.3586885135904583], [0.3269119633050932, 0.041996675553737345, 1.0179185743348893, 0.7200408152969097, 0.43418096531397143], [0.5012947165049102, 1.7637886662973092, 0.8607970195904383, 0.4503721393162093, 0.2406441973007049], [0.275746990644566, 0.8210640284029531, 0.8881537419613229, 1.2417838988732686, 0.451572461620799], [0.5928396905366613, 0.06223673942537858, 0.3945790782761639, 0.23461712589089811, 0.2662411170477009], [0.47295132655824523, 0.9216281557749019, 0.6455957989747543, 1.9213980542023408, 0.11667833941260675], [0.7636089972195848, 1.146847595843898, 1.0306215080698848, 0.6965327632060765, 0.14008100033909304], [0.106314172151322, 1.285901287736443, 0.2992857351346626, 1.4150617765966178, 0.7893166684000665], [0.1893251164332621, 1.6355859357345348, 0.6725583158366977, 0.3861225901341032, 0.5298118327179752], [0.130112498756418, 0.0020478372416982034, 0.12873472560766683, 0.3132763528130201, 1.1345851468749326], [0.4872999727263112, 0.7321702575083213, 0.28151934798982403, 1.7318195833400387, 0.3326589677617816], [0.6581436500323434, 0.3010392135030857, 1.0896659654852603, 1.061462285472158, 0.6156141968786463], [0.4091252070490077, 0.18556682619785925, 1.0825156540252303, 0.9141352163448115, 0.7071546054296542], [0.4624149382821566, 1.0286900705798927, 0.058963373031979405, 0.9005287008187816, 0.15189432836846406], [1.0711373959716164, 0.34514748647773463, 1.5483120231001815, 0.6992741029457223, 1.1934303987841952], [0.16563546403841212, 0.7078092329315777, 0.12715934562061765, 0.2915161556374284, 0.2501941389972409], [1.0698509647068675, 0.6076373054710859, 1.6326571875086457, 0.37236429197832127, 0.5004141220595317], [0.21324977718108457, 0.9381253599296276, 0.6853602205486983, 0.8785656716540928, 0.3845444943966638], [0.03134100412538616, 0.7990432013017957, 0.4806676531251987, 0.3789337494110906, 0.6205865165366569], [0.5247669426110264, 0.26893938012931057, 0.2212316040691108, 0.6796270027800901, 1.473018380414396]]         [[0.1399876594051076, 0.36260783175898276, 0.2838110155045018, 0.06368226769105607, 0.24447745159387738], [0.11001492780549715, 0.8430610169250244, 0.9780995659839793, 1.0076449228936353, 0.14356998797506515], [0.12168188018834103, 1.447685758479953, 1.0815594670892357, 0.458230068288146, 0.28621643605228064], [0.2344172238280755, 0.7224614025377092, 0.19401205338758049, 1.280214808618561, 0.13653671692161384], [0.039798813173382186, 1.4291980262036357, 1.3944110726077532, 1.14199107075834, 0.2355155794859778], [0.09686747548724511, 0.5507241212643579, 0.8315525193674329, 1.0046380368626542, 0.29277923075091566], [0.21582076376810155, 0.6447137147827771, 0.07473823623818818, 1.1497502360206864, 0.6970143407700181], [0.2548803789861981, 1.0336263597134738, 1.6544708835964563, 0.17588045236556446, 0.31625715868651727], [0.1336211793009663, 0.6203743936679564, 0.6191651446000315, 1.2087809605419442, 0.035131213791745663], [0.2933090969626272, 0.889402588326965, 0.7930929305112757, 0.604212778908201, 0.32579535635021095], [0.31447513225739, 1.1342385192593412, 1.3516275080987028, 1.4658579462507393, 0.2113073304254065], [0.4126081070163914, 0.42597001008194385, 0.9600009015760383, 0.8996357455453411, 0.7126996821910826], [0.2942047080356778, 1.6476854030910488, 1.2267389804894007, 0.680324592051273, 0.37684455869163574], [0.2836504886074884, 0.8296731580518003, 1.4257121710175085, 0.9911263540326692, 0.5440466777239339], [0.1959950037715133, 0.004052969422980423, 0.5330279591455517, 0.17996166615922, 0.4094986468764825], [0.02701439583354004, 1.1285129756083352, 1.4117352036140347, 1.4101849547054965, 0.044336196994174015], [0.06267840856984291, 0.8830885534364878, 1.6674298743533769, 1.0846175448518414, 0.5019840271497907], [0.1990550399032296, 1.182874621769099, 0.10069752086296277, 1.3850200003465347, 0.65200555718517], [0.062166342943028585, 1.4200344007910353, 1.4378164740953765, 0.8563700363077094, 0.4025850516458057], [0.29163956600329993, 0.4099974001237859, 0.5517309912501648, 0.6993630206622061, 0.8090884336856847], [0.19267921079016737, 0.6164475425168114, 0.9292396131425995, 2.0607200080559758, 0.13549259611660036], [0.2532173011455898, 0.8519168415030894, 1.8993925266759475, 0.44076352679085096, 0.6551574690415961], [0.39969892562510756, 0.16335753940973485, 0.6747685525151947, 1.0884927154985258, 0.6267514181306104], [0.1593812822620222, 1.1750709588616608, 0.4086744140155313, 0.6445237120316041, 0.27534989631188933], [0.7033586616475, 0.4327484717190157, 1.1521078706857024, 0.9692533264991801, 1.2962782915644981], [0.25465060284530194, 0.27011013958595737, 0.29510002415934006, 0.524204589285101, 0.3727590653974998], [0.47560376009939365, 0.818556263505843, 1.259435645902084, 0.6758065213248085, 0.9014604398417665], [0.23061712258935207, 0.09455427800973301, 1.5955157049175754, 1.7596214934030932, 0.10029184979765127], [0.3275066272652589, 1.1751548428122358, 0.10127855302559556, 0.03609333559426124, 0.6171977175256523], [0.26046546776696644, 0.774242358599326, 1.197489683682925, 1.583538230794752, 1.0248047456045104]]                    0                [4.678072448533853, 6.30261420638153, 5.147342832542393, 6.77549846179052, 5.709026785750952]        [2.340541335732881, 0.9455260543020081, 1.948213980874046, 2.1258386339067497, 2.0000771863146856, 1.013657549323158, 2.3756085210137607, 2.4869607996126852, 1.510192013323325, 1.2803612430567841, 2.257803930329002, 1.6549030993423952, 2.121398165364291, 1.2455718501375288, 2.3643934865553384, 1.8613865832614302, 2.363160006373977, 2.9065910161077424, 2.1227827620423163, 1.7656639382971986, 2.4554639381518437, 2.5142208646377053, 2.0333208065595962, 1.7836590690597804, 3.5990936446105413, 1.735047908544193, 1.8857164828681185, 3.413207999846903, 2.6446145882845635, 3.2163745441219893]   0\n",
      "[[0.008997266421676363, 0.17191993269127218, 0.32354487559275413, 0.4410776556218223, 0.3835749997887506], [0.5301431516466155, 0.39424443736021986, 0.9255658482363125, 0.9822050380662809, 0.07083918961788792], [0.2507050419213873, 0.6955224533848507, 2.5271799415475336, 1.316859281134648, 0.3689028983210947], [0.2662460673701382, 0.2264382098919644, 0.7484223542940507, 1.9470774473254175, 0.4358824392669054], [0.5154013598335463, 0.9902096891207505, 4.833595959397763, 1.6564216326968875, 0.06713595177719878], [0.40018954053603695, 0.45014958635840707, 1.3921623014183262, 2.824154215510394, 0.12926789854220794], [0.6035375896516544, 0.10882983940122148, 4.998884770427768, 2.4341542409378687, 0.020657169001852438], [0.5524192999340634, 0.17309346206766585, 1.6008445894813614, 2.0959812849868147, 0.30575151520157107], [0.7465227698129355, 1.6472192217260673, 4.435955205752009, 3.3095799178656673, 0.5172455591329689], [0.48459033949979935, 0.11809972317381212, 0.08240395926250058, 0.07819339958598687, 0.007736261406553098], [0.553660258258263, 1.7126677114480935, 2.8001031377779477, 3.3220500184225865, 0.13847193858162976], [0.9669029442969337, 0.23926722714561033, 1.6146784995507324, 3.0220086891008715, 0.09785241106172127], [0.1815543154977095, 1.5923338903749877, 0.4924542163173846, 1.146165925306955, 0.27295602158994164], [0.9303652367122814, 0.05530186110430435, 2.9025824298537466, 4.6966689508992365, 0.28812927331412796], [0.4926296773487684, 0.5749786086212325, 0.8842345554030321, 0.21360905392401552, 0.8141883295296026], [0.7096880706851556, 0.17672486552605493, 3.4041639388551044, 4.802335798018895, 0.346523197616212], [0.8303941209564922, 1.0673094823550502, 2.266171725687591, 2.768139136415609, 0.7902646434854766], [0.4382836938695007, 0.40835501177431033, 1.7192057525274262, 2.657483241614239, 0.6530148852726918], [0.7096266273143377, 2.6236174713061873, 3.342941509391687, 3.3372140297707613, 0.4739594654019419], [0.13880091364573305, 0.2741745775967188, 0.5278861232890248, 0.22170922278269337, 0.6831567738027354], [1.1293535862550113, 1.4688032817605767, 3.614200218493048, 2.059544648666603, 0.4342661536040119], [6.002025312543274, 0.10132902310854186, 12.870802348589443, 4.864709083089961, 22.970365362292956], [30.228536535021288, 0.5350850672141079, 1.1975808595293307, 26.636084359149724, 1.7605476835018996], [4.616976502033501, 27.55262317979518, 10.791428469502346, 10.668042397163447, 9.831160376170427], [39.38982819114147, 8.600898516379113, 28.18364019135123, 11.117604813863267, 3.5721164034837694], [23.716505031216172, 5.168404612483674, 25.30299485217888, 14.32982890801343, 17.25707579289764], [4.080757958264736, 29.574724013009682, 4.909605099521663, 11.357123734806885, 23.403344307296596], [17.297603077023965, 6.6848031936740515, 9.271300643618826, 11.576528709215692, 3.613496768885433], [7.666886168193194, 2.965960572288557, 11.284438219153689, 8.703502775554238, 9.984441991558827], [33.08518077461325, 22.956444883627665, 31.996586108480706, 18.40252130129429, 5.756952313182498]]                                                     [[0.17730375341870874, 0.20251778823480815, 0.36846401378702104, 0.3783950810656052, 0.32040082186484037], [0.3108380647874398, 1.4489123044991237, 1.3835131489462562, 1.3342038526838969, 0.11352663249501549], [0.15191235217145482, 1.0546423233474491, 0.8563910966222407, 0.32278963857465554, 0.3243224589606366], [0.058189447479650935, 0.588767746446806, 0.05802262613629433, 1.155472975987291, 0.15310037831477064], [0.0828778158051755, 1.6851014334169154, 1.5902516917818454, 1.4766360963407954, 0.0537027126951779], [0.012947330248178512, 1.121216426861614, 1.016893315365621, 0.4840999742930635, 0.08338414555510781], [0.22855009283653915, 1.2454008131221341, 0.3178400266174467, 2.1067377697345866, 0.020177895472467733], [0.33041446613201403, 1.2859647525385913, 0.9391790404841599, 0.30864813898593, 0.4293928614001261], [0.03650087001255797, 0.36681286501874233, 0.6923600679445572, 1.6509601854228104, 0.3427076903907521], [0.40182807048158276, 0.9867941977146689, 0.8763493588195009, 1.0023541142247072, 0.44219696571169115], [0.24656719872318655, 0.5770069224984594, 1.506094018166633, 0.8502154738890589, 0.16705610463098142], [0.3543914307599765, 0.17527864737570442, 0.28066944604331656, 1.1759031301352383, 0.4740346283896312], [0.3625100963655309, 1.095064877645371, 1.527000187834088, 0.8087663790376795, 0.4272154759883942], [0.06069521845104475, 0.7531299191525743, 1.1122577803252947, 0.8275799933381038, 0.1428697328473354], [0.3165445549676086, 0.7924041733357365, 0.10560482724033762, 0.5274429315783511, 0.7663656931598769], [0.14222114915629214, 1.1832905807031626, 0.8532052671719104, 0.6756187172740455, 0.28141489384886686], [0.27979465940340353, 0.19064402271043476, 0.9975108329174549, 0.39868988535291233, 0.7429899328097984], [0.22918773846051932, 0.7657873053430623, 0.4812427501919057, 0.5443032839637166, 0.48549895693145906], [0.05133523541960194, 1.5763198324822054, 1.03306452676393, 0.8967234520789994, 0.3382669314559906], [0.11397049024815714, 0.17511867433488593, 1.1185885755483116, 0.3319836477849799, 0.4287698057483613], [0.07096579198501107, 0.5172131752091726, 0.6400548442143728, 2.055164485845724, 0.6308212338146144], [11.88134702620492, 0.9632191621289492, 15.242463400108623, 6.8595117930425, 20.41337554801046], [4.34470885590596, 0.47047881554966386, 0.6511847774522548, 27.03625107877714, 14.505614933810861], [2.520577610727381, 28.185835110392944, 13.424488975996615, 13.118817002299691, 10.5686731635773], [7.415656331874352, 7.3560662370731595, 32.24765239522805, 5.398263557958315, 12.680957859499063], [0.7110745450960785, 5.595852183794286, 27.920164691564587, 18.16432780156037, 5.818177349327282], [9.354500339109649, 29.43850064061574, 5.540625269232127, 11.231199675282708, 16.875800179545056], [2.691445946368791, 6.093973940154101, 8.867417440029211, 12.789691792957473, 3.2969327714939505], [2.0026076687129812, 2.3379236678987905, 10.233833549886928, 10.25262520583494, 4.957388497540087], [7.634465445371383, 21.35861161610668, 34.29495066563298, 21.98668072914242, 7.619422075160456]]                                              0                [237.5421821785525, 159.61094598909222, 193.75199420628272, 141.43792123815405, 139.03333098836114]  [1.9023715446095135, 2.7433491971519794, 1.5062920724849695, 2.0977378111206115, 2.6171133988921067, 1.9879617135270036, 3.5150573496102027, 1.8711795377253642, 2.289848986836407, 1.6733306161940436, 2.2479661737406897, 2.110098937506398, 1.922180230475436, 1.265904383054202, 2.062671044771529, 1.2600243096882338, 2.1707200921112824, 1.0895694767529915, 2.2906958212740665, 2.3724842781406124, 3.476585315068946, 87.93282296312971, 80.58809461755129, 76.95450624438965, 82.39088923127072, 68.78124568450053, 92.77443732151438, 35.14621906075952, 32.36660128101787, 99.53221453908209]         1\n",
      "[[0.20891761199925626, 0.3548267350271715, 0.02238133071784914, 0.03043722652152188, 0.385542006693887], [0.250948416067711, 0.013922534229348804, 0.7386984900887619, 0.6203818327395527, 0.9051047485118011], [1.3898629704291938, 1.2795394547215098, 1.8292167892668945, 1.5106435178652013, 0.8478996858338352], [0.6109516421509247, 1.0069411853211085, 1.023694892334448, 0.7725358501803143, 1.5052218436612899], [2.1586477501845764, 1.6022298241142663, 3.5672705090540786, 2.857853756356456, 1.7776052732465444], [1.4006153256434781, 1.2004558570328503, 1.102874505298054, 0.08028416873155708, 2.0781671506947332], [2.532047103370076, 2.2553817752969056, 3.9086224710146515, 3.8361334021023383, 2.430596220923784], [1.3014706800841267, 0.6311941882538192, 0.9045444838760395, 0.2114510408510175, 1.7309231855869918], [2.4955885715691313, 0.41857175613807973, 3.0774924248919198, 3.6642729300305166, 3.5910673624832974], [0.11946033180837676, 0.5510206447585608, 0.19804669791609106, 0.23596652247850727, 0.4336613685413166], [2.5910614251952118, 0.18910006720923211, 1.801117611577243, 3.27368681446306, 3.4268706575210093], [1.276635806403494, 0.6186948990920382, 0.7530083276296426, 0.3479732588854668, 1.096049112808554], [1.3553629794068631, 0.7569112411235088, 0.7754015524006495, 0.9562995855903482, 1.3519400744520083], [2.611654308864723, 1.988873067286298, 1.1883364613311096, 0.8502280014334246, 2.6566013035930225], [0.40048569707228765, 0.963492645676801, 0.024764641644906993, 0.06370327008833737, 0.18322186263998452], [2.787207573513514, 2.3988523707647316, 1.41526477352392, 0.841725508374745, 3.3572164341461557], [1.655082518626703, 0.20480219729737015, 0.5174078773615083, 2.0383846319292713, 1.2679200826507875], [1.6422963694305144, 1.1016635730063906, 0.7589260932970048, 0.30871853167807006, 2.5167467978816003], [2.243501066899484, 0.20105569266519818, 1.7035032522154159, 3.0516910652661133, 2.6437215132613785], [0.24271217239086162, 0.659352001397997, 0.16999095519795424, 0.5379672774409605, 1.1169567838101562], [1.819797675764525, 0.2885215851259334, 1.8297544242764472, 3.952815466956758, 2.568940723014131], [1.5975366888108031, 4.759456183472207, 10.545834011318274, 9.183148091959163, 9.398757825871046], [19.855136634074285, 1.3922061905685736, 1.4668265547151078, 24.77263197576111, 4.071829623957766], [2.0762265669624034, 28.94489792430176, 18.66591599745064, 0.34367035967261894, 16.26322815104568], [28.633099745583994, 5.96482580689875, 23.20516505766583, 7.5646349980280405, 8.999592921010015], [15.307293329771744, 2.9186535530167856, 21.666942911398007, 18.346581315046723, 21.64919798520486], [6.53690547410493, 14.083434892815465, 2.228963780191455, 25.968342053823097, 13.845546870050178], [12.60672654625206, 8.224889804938927, 20.520690746692345, 7.941928356195343, 2.5479180827836587], [15.009512627014516, 6.703972201066839, 6.41920738182519, 8.321212365321376, 14.75692759699853], [31.31970994145971, 12.32254692700142, 11.899569807792547, 28.24145700187062, 5.645742196829322]]                                                                               [[0.09650156657968634, 0.336147089040892, 0.005355747608690883, 0.03714864777550417, 0.19936430888257553], [0.2677669732097545, 1.3685670768848686, 1.4688430638917702, 0.7638217645315433, 0.1449667820548457], [0.003423672986829329, 0.8669253676683297, 0.7150809389917206, 0.8065747346939014, 0.12697889044936328], [0.05568348653126165, 0.05046034433406783, 0.7359666089752426, 0.5645500231113174, 0.09409026072222793], [0.05310855881458765, 0.7739581869591947, 1.4271365829253033, 1.360476897247615, 0.27613995865235297], [0.22340885931882226, 0.8291049651898137, 0.537721109986606, 0.922974463157152, 0.5465159902320433], [0.12740574203745414, 1.2718236689384363, 0.7789046581369434, 1.9156762217079983, 0.6368411920283992], [0.3983342128034323, 0.8677197197384572, 1.007096787723749, 0.46442030081767194, 0.7420361436121982], [0.19728803557054242, 0.6268459643363193, 0.3209038632359492, 1.5078038736253898, 0.06007631365115096], [0.3800995194768326, 1.3402320026739336, 0.675805803702711, 0.8904867975825177, 0.4293695249356002], [0.5161628237496476, 0.5803673769219383, 1.1012352667959542, 1.6297635125379855, 0.5795749238250815], [0.40443977596415603, 0.3924893391143785, 0.37522855193220156, 1.0625458278492457, 0.5985881454083619], [0.09488980398795621, 1.1747067583794082, 0.5720301513585866, 0.1199435974867061, 0.15230963916346807], [0.04023543707256794, 0.8809805897605225, 1.4425652067917376, 1.102197279485237, 0.04058567773189714], [0.41178120494995507, 0.8299163872311006, 0.48238435996694706, 0.4073834322393125, 0.8206416295028766], [0.28743489125280597, 1.4636099289625266, 1.4270243680829058, 0.9435418041666339, 0.5520450977397986], [0.18836278914333104, 0.3459206324896577, 1.7034607531163155, 0.8567412304327442, 0.7306212581401247], [0.34917599063129123, 0.3528136033590605, 0.637241623854001, 0.9443002306780022, 0.811354325160481], [0.01634822164778016, 0.7577057122455623, 1.1889606487531725, 1.1558503383991205, 0.31328154985060663], [0.3852994264610492, 0.306992384459824, 0.6900575577131933, 0.7965422386547799, 0.9993044577618718], [0.07168984346918039, 0.7454839154442375, 0.9596112821813825, 2.0038819653582087, 0.646155719993092], [4.364850933299208, 4.8326963798311695, 12.73573311724895, 9.186730615252088, 8.311423873061525], [6.136112669399935, 0.1047225078828159, 0.23477659572740006, 28.111242125319446, 6.118763827507391], [6.717603857380768, 29.347880588254927, 20.885121873945167, 0.010588944579853377, 14.33155604248396], [10.29570048028462, 6.936532401408017, 23.624020150638977, 10.673994711680288, 20.447769690122833], [5.589489651064341, 2.4301773720614244, 19.85558666140021, 19.250704252346722, 13.17748935850602], [9.169236123396933, 13.995937343850223, 3.2849197385628455, 26.172829290706556, 13.570825858086925], [0.5356358895005009, 6.73429339792069, 19.210387172993276, 5.330907725340676, 4.625312623697789], [9.399684223084853, 6.234960966930079, 5.646999520447232, 9.219625362465338, 17.592267220185928], [5.645164628279279, 12.82801835530593, 11.414259638055395, 30.020060240665266, 6.581396534881257]]                                             0                [185.85151482234434, 118.70376972268397, 156.47231950654572, 185.0769863358365, 124.92159794168484]  [2.7563052198791143, 2.3611815594402663, 1.5944085596202033, 2.2619790195622334, 1.7552278326099218, 1.0091808146161731, 3.1492641093031293, 1.318630237350014, 2.177839627994287, 2.017440871051296, 2.9701514490481875, 1.8448727169163774, 2.4252341801065964, 1.7234525680270965, 1.659051100426757, 1.6264622722734103, 2.893857006421137, 1.7425436062705564, 1.7306707978020996, 2.291165999007744, 3.3180348169266805, 43.105048944699185, 73.61734135692356, 79.66658796052194, 90.44852504712837, 73.72083409088813, 89.49431270548502, 41.82721551159135, 69.71182262906709, 78.74188247780664]        1\n",
      "[[0.4742415323107961, 0.2729328058610238, 0.43546386383209273, 0.3778658320426913, 0.19649667616941133], [0.17401720794983178, 0.104755633022683, 0.7141567862039968, 0.2187331660399095, 0.21589642012208288], [0.10221796123045884, 0.7161493260868453, 2.690637385860576, 1.2792406375068712, 1.4712984622836878], [0.042968922328092296, 0.247143304260757, 1.1440132478913183, 0.32195261600602776, 1.1909877141077874], [0.8860607905273212, 2.0465730160054796, 3.9163626437844785, 1.0863623321760592, 3.152134366180613], [0.6486859647166854, 1.0825765508139789, 1.4081570882343946, 0.031704948384810555, 1.3846854676495224], [1.0957991705488563, 1.1680553319955218, 4.981384090599736, 1.636500027158467, 3.91625533062589], [0.739137991189099, 0.6538241009370539, 1.3307225024634664, 0.8946731196717367, 1.865984881772898], [0.9425526408687713, 0.09647425227022827, 3.8633878643515374, 1.7000628038936267, 4.539460734006806], [0.24757108949204382, 0.055539535463935856, 0.40858030640835397, 0.16918032804126532, 0.3302457020972138], [0.32156050788732216, 0.9836247977413344, 2.8001219158714488, 0.8970737598615826, 3.128911481207733], [0.7534548312853917, 0.3083312590908879, 1.095350458211459, 0.6078020871924771, 1.8597315046553053], [0.8180165735468277, 0.6774013280637916, 0.6466506476622502, 0.9147926730302331, 1.1964641509595677], [0.2216615112755823, 1.8556599806110512, 2.615051576357126, 1.1307761923121495, 3.8320891507655563], [0.21161573239119322, 0.3994765513575476, 0.5148040938699169, 0.5386472502304909, 1.483273472011874], [0.5856524633283844, 1.5850648425826133, 2.0804028398495094, 0.5964590008755781, 3.271167866257843], [0.5015816219540206, 0.6070005710557089, 1.7814897573708315, 0.9510472297200219, 2.858222021197272], [0.3417793368325306, 1.3634552624165142, 1.2821670134043084, 0.7594043647692488, 1.9281032782024439], [0.5710175518317702, 0.6304205907936429, 2.058138733146349, 1.705630965894474, 4.292099059118428], [0.9072398937761035, 0.7551987267106841, 0.6092684449875558, 0.19465755891499692, 1.1245170455973872], [0.8285514835250055, 0.30441135268262187, 2.919342171998089, 1.5086595168638255, 3.1002090288091746], [9.515331687854928, 9.05356010887206, 11.431930353351577, 11.334672276208702, 20.942189678095747], [12.19673472391636, 10.416853189458823, 0.7269256972186957, 25.38754257677281, 4.329357595980065], [6.184417275418065, 25.396159779856163, 15.44683693608459, 4.044368509516803, 19.566396402039377], [22.325929438194606, 0.45269934902146014, 11.991351894547108, 11.627863809764786, 5.844732357234383], [27.927832901005075, 5.715916532264459, 12.048635052169445, 14.251832561119478, 23.608223489616165], [2.229058424714748, 10.348303738474963, 12.728440143758327, 14.48752958618449, 10.898721137854315], [16.303022010440174, 4.321158855265036, 9.91371525147721, 11.561884766926815, 4.463759430633406], [11.599146081231034, 5.771343960313821, 0.11159176071507423, 8.18761924483643, 8.31136664489828], [24.783708462408505, 19.724909369512673, 15.899686700084569, 42.2089821034099, 3.87166308971349]]                                                                         [[0.1284835445989977, 0.2909773060595233, 0.4507245003509237, 0.380950319419271, 0.3998954552410958], [0.14007669240795062, 1.1814708850796545, 1.2822340828093188, 1.348826429397888, 0.20585828277138418], [0.07826824171948937, 0.5251444122060488, 1.2190768918535564, 1.1162203295992388, 0.2853006233898675], [0.29151191144275557, 0.5947795744613373, 0.4928415058325891, 1.0883606146225036, 0.26414814859280056], [0.030019053512247984, 1.6882829804345518, 0.9899239749126272, 0.8157400775766819, 0.20187922725574803], [0.3141812735802464, 1.0313229896811957, 0.7686422456960684, 0.03701106667062362, 0.5016925870673408], [0.13255709021732875, 0.8381184790317234, 0.7563581129417899, 1.4270836150944677, 0.001833875502617932], [0.16635734517559947, 1.1712807914909988, 1.0583426977593322, 0.3935730445750213, 0.2466246702870641], [0.06579153226385315, 0.420279366141479, 0.7517806827257052, 1.4537256355323418, 0.2461789875576731], [0.20094812745559743, 0.7668396486506298, 0.5314031686076993, 0.6017447347306847, 0.17704506712747653], [0.15618962784272972, 1.1762726268863064, 1.0619534263037038, 0.7438373578381963, 0.08597824475979132], [0.08033540285470431, 0.3673954924148559, 0.5954186194616802, 1.1390770228354923, 0.03658674360368985], [0.41117207131665034, 0.8162845498061329, 1.1660030850230196, 0.8024959838613804, 0.5525001233937754], [0.3223655497975464, 1.329081963438227, 0.9799900157014356, 1.589480926217249, 0.46543649518452], [0.3285074112479141, 0.3800355880088274, 0.1903118027664313, 0.5125455426560213, 0.801044132473831], [0.08603776157934262, 1.2395159909915996, 1.7422275822213873, 0.8600150375235327, 0.10921750402507191], [0.015497462039700816, 0.7222047845264776, 1.1654105559528014, 0.898871963025442, 0.25863387293978707], [0.053515647733974214, 0.9563591624905827, 0.686513419828247, 1.118296365271323, 0.01625371689931654], [0.09232142995728898, 0.9727613304505388, 1.8678926378857128, 1.4251253304338263, 0.6522439257110211], [0.42132427420317486, 0.38528625068969874, 1.1490221517714279, 0.08259514105601894, 0.9641225371128348], [0.315877613975853, 0.7238198018025138, 0.9035430353975806, 1.1787206416514984, 0.30685618998201536], [6.5294029287415505, 9.143177343114226, 12.97465637467321, 12.813899409942465, 14.71561950113253], [1.8065533993013343, 9.715661410664028, 1.5685378675791655, 26.964455649526535, 1.6099299546419754], [10.548126447665801, 25.691893694184834, 17.630861901735447, 3.368342194569887, 19.44403410424004], [9.29255484730824, 0.5277723439642468, 14.733133788399993, 12.853435099273556, 14.053986034638925], [5.69923839809821, 4.274929162141448, 11.312783475747745, 9.996086000449722, 7.737663983425868], [1.4092979632892217, 10.467464125193969, 14.156439174839345, 14.896182311673659, 8.032120685599159], [4.602153262928097, 3.459386842000034, 10.081056026427497, 11.338324739478153, 10.323419225683091], [4.992126802104, 5.555665875187691, 0.24445484100498321, 7.820570296900165, 11.38252340932147], [9.710122829952848, 18.99904934675786, 15.970124991635409, 42.887960023637696, 10.96715978838333]]                                               0                [184.877319463517, 118.89878699451953, 112.18970084785585, 202.99476042785022, 136.48442534958428]   [1.8559008973641555, 1.4110950451544897, 1.4681444631889848, 1.5714336366523403, 2.5386026961610137, 1.8757250120065507, 1.701028111626896, 1.6081718155314795, 1.6333305087762033, 1.4800476961189852, 1.6523715581751115, 1.597738118897247, 1.556463187395471, 2.010823293653581, 2.0360816408186877, 2.156256974608611, 1.609280828550005, 1.5327071744036986, 3.0411289272811524, 2.5809349865618483, 1.4535132636147048, 63.80791082749012, 59.552339455307276, 89.20781258847221, 75.50873794722268, 47.95638828837181, 52.29033376409495, 47.17204513365078, 44.07135864382392, 120.70542455581268]       1\n",
      "[[0.461209940391607, 0.272716614898264, 0.006162989258057691, 0.4953713559247196, 0.4048628521032126], [0.7067971557116957, 0.01811439443274495, 0.01814733270842303, 0.0039624969960593, 0.1424938702825606], [0.06930513834344165, 0.535288766223991, 0.7698140990051601, 0.08314460908935861, 0.5812826691080738], [0.21562840123758892, 0.22873773507035317, 0.37067824701811364, 0.05869025927058408, 0.6770646889038272], [0.41605403307361555, 1.2171716961388985, 1.734584947987272, 0.6384476304916202, 0.05245504018900643], [0.3718405960032314, 0.7261122706076906, 0.7534356501528088, 0.6298251842421654, 0.5140987871987486], [0.5543676420335897, 0.7340624005208788, 1.4807298910239113, 0.7456338941177639, 0.5548673000399568], [0.31059878591108747, 0.8143142519924328, 0.23994610528926213, 0.45769101066134305, 0.05834385184265717], [0.12002712591544618, 0.8107293561789085, 0.2172354050062818, 1.2486610514154437, 0.46546324625596713], [0.3945539801137131, 0.20428725513294035, 0.06023629596334296, 0.3512108083234044, 0.18110427936127477], [0.38441521565103276, 1.83144747166798, 1.2509839812533867, 0.3926177668503821, 0.23177818851969556], [0.1784660472183749, 0.3498470177789412, 0.9483008370105644, 1.3500897453774097, 0.5755973534118075], [0.02300232274074987, 1.146914163173812, 1.0443018132115034, 0.1467542142660032, 0.6684300769502314], [0.021745190261236447, 0.5007854148403393, 0.8992229234722188, 1.5484518376636411, 0.023767492410007862], [0.36220445039489274, 0.34125739001641137, 0.4261288202857474, 0.2370546243565242, 0.8853559562592388], [0.37400118376871183, 0.86204027856148, 0.7392181312915975, 2.1737634451298886, 0.6122539563753407], [0.10266236427909692, 0.8076131999361824, 0.27640848813581864, 0.3146335184605338, 0.6612000184177378], [0.09569480238089234, 1.2153790167185268, 0.17786748300746227, 0.813599351515846, 0.9955838110005967], [0.4915266154181275, 1.5297635163547383, 0.40704023707078973, 0.5865861217668702, 0.4746192528736263], [0.3018986200561539, 0.03817401361597027, 0.3804195534294197, 0.39545675150491333, 1.0358865425647896], [0.16814873518515872, 0.8172208773992623, 0.25617952713841374, 0.7730337740574453, 0.39845443606851083], [0.6115967840929363, 0.410444462974191, 0.22352951663081355, 0.34906344708850257, 0.5903296776065925], [0.7511720287167212, 0.6599309333090666, 0.5036627810022264, 1.4196037284351073, 0.32086733701245906], [0.575535419346797, 0.6664706832628025, 0.3304397850235745, 0.8782286722271517, 0.13436013499487653], [0.8320349113201454, 0.3464321757534034, 1.9336930803553576, 1.1995583397729535, 0.6961787850688834], [0.017432328246951845, 1.3921789339178325, 0.3226981859333362, 1.0472483918958775, 0.03095540086097981], [0.30170123796442594, 0.6153571057238454, 0.9827758325434801, 0.4503138979202626, 1.255586117457491], [0.6944804695848982, 0.5106451394575099, 0.1984874509889445, 0.7057709601281057, 0.058021455255495194], [0.888529211767098, 0.4242181200785949, 0.39824306549948646, 0.16023911058461346, 0.9182787020820583], [1.059664423729159, 0.06357937877169545, 0.5210209916040395, 0.13725361965478908, 0.6837978345021376]]        [[0.09337611735170293, 0.2591430469796518, 0.00383763180304339, 0.4762255133480914, 0.09500107098400717], [0.21997804254932918, 1.2074869761289644, 1.0646511209146619, 0.9195651986817652, 0.06810705526585314], [0.2674945217334571, 0.5579321038778657, 0.572666470493938, 0.3161430274535373, 0.5615708508356514], [0.03279705243708819, 0.4391281554313876, 0.14790045130170593, 0.5487613070012981, 0.21921386292919248], [0.1442828979341817, 1.2555084094395736, 1.3789931099570072, 1.030853531044945, 0.1602676878774213], [0.1820143646957456, 0.9341527397266746, 1.1589371644511701, 0.23357814797033666, 0.24281785097394215], [0.25538055614916044, 0.9455395408348599, 0.8392648725773472, 1.4151467843651833, 0.7621485859945037], [0.3007476741380517, 1.5566328809661245, 1.1331684955366303, 0.40580655105347185, 0.3161001120223874], [0.04592110857363574, 0.5318711374728519, 0.5258099285696607, 2.021819621822721, 0.37815927585119513], [0.25722243969749864, 1.0459116259963235, 0.8991354390194759, 0.46021502316538465, 0.26644323357693117], [0.016350052972524898, 1.5256075431893477, 1.935573836010594, 1.091558187642662, 0.4121994131096779], [0.4046298891073359, 0.03951320860716163, 0.8245117058359765, 1.3880072261313863, 0.5940527047373676], [0.36153611218974835, 1.0570102584001273, 1.3015180096681374, 0.3913929929224304, 0.5382446228374352], [0.06028479266814424, 0.48069854278365853, 1.2728458292974003, 1.1388043607041267, 0.03600049198412725], [0.38224849034834507, 0.4116957660087073, 0.5582019825091811, 0.10955098444751449, 0.8157422858448518], [0.23520653542781256, 1.0311422558961432, 1.3181726302569428, 1.5445427762224357, 0.608962894200641], [0.17978906490425217, 0.5481441643845786, 0.8334929922162991, 0.9024237682691834, 0.6187328011208132], [0.30068424567166674, 1.088609597317342, 0.09143256248756654, 0.7268429368562245, 0.7154207508034274], [0.164133302776859, 1.3576800367930535, 0.9686098907866649, 1.1554378010396475, 0.6207965534823091], [0.26951768238592505, 0.44437912383188344, 0.7843396241150165, 0.7683756565790595, 0.8071021513338396], [0.2634271652357374, 0.7185275393638914, 0.2538128871031412, 1.3251471186370412, 0.24230483001010406], [0.20176688077605154, 0.9456995498209119, 0.9455477618208604, 0.3532826981354757, 0.6232138791256545], [0.38241046762838815, 0.6979666017645445, 0.21077466901777553, 1.7595390046842712, 0.4353629588936674], [0.15594680744642853, 0.7913932541030778, 0.6854455819265647, 0.5350568527066297, 0.30538813516153873], [0.4881664159708663, 0.41753341117749243, 1.6287849189051198, 1.5515087949891255, 0.7963854059355837], [0.22646479817528387, 0.9377424232415041, 0.07300678103218108, 1.2313896337563706, 0.23477301725998423], [0.651335233682551, 0.7964544281292144, 0.7102672987376148, 0.6970435625439857, 1.1866883049005303], [0.22798482126496555, 0.5156869220147887, 1.107029463975574, 1.5115910469830856, 0.017217691336260277], [0.6431317416381992, 0.8115084489203237, 0.005495829153780104, 0.24656558316476118, 1.1880159847081044], [0.11403825163390913, 0.9651235206328068, 1.4643046452022295, 0.6883859174046286, 0.5789518184984166]]     0                [5.277801826786314, 5.690847682217466, 6.210177945808102, 6.972855807027882, 6.2229681134913974]     [2.4197900813306092, 1.94786613808706, 1.3622334511307665, 2.1965419660050793, 1.5440503691871958, 1.812480575249519, 1.9528957856291547, 2.3228726812578593, 2.875405857697659, 1.549507853198584, 2.9758493344123003, 2.5627929209458666, 1.6679666093312895, 1.7905537332281565, 2.0482780991873386, 1.796902313501064, 1.3924217821596512, 2.2694252158219532, 1.9971236250806053, 1.752128886956888, 1.927484501796512, 1.4827236744056935, 2.751944464490283, 1.0532083705001134, 3.2148288025626384, 2.2084392306055878, 2.8123114868464576, 2.375074861092791, 3.3088411075591946, 2.007878524556524]     0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: The `gcloud ml-engine` commands have been renamed and will soon be removed. Please use `gcloud ai-platform` instead.\n",
      "WARNING: 2019-07-12 20:43:06.126365: I tensorflow/core/platform/cpu_feature_guard.cc:145] This TensorFlow binary is optimized with Intel(R) MKL-DNN to use the following CPU instructions in performance critical operations:  AVX2 FMA\n",
      "To enable them in non-MKL-DNN operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2019-07-12 20:43:06.138637: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
      "2019-07-12 20:43:06.140533: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5571a9798fe0 executing computations on platform Host. Devices:\n",
      "2019-07-12 20:43:06.140580: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "2019-07-12 20:43:06.143312: I tensorflow/core/common_runtime/process_util.cc:115] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n",
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0712 20:43:06.143727 139858127025600 deprecation.py:323] From /usr/lib/google-cloud-sdk/lib/third_party/ml_sdk/cloud/ml/prediction/frameworks/tf_prediction_lib.py:210: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "W0712 20:43:06.286077 139858127025600 deprecation.py:323] From /usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "2019-07-12 20:43:06.342879: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "model_dir=$(ls ${PWD}/trained_model/pca_unlabeled/export/exporter | tail -1)\n",
    "gcloud ml-engine local predict \\\n",
    "  --model-dir=${PWD}/trained_model/pca_unlabeled/export/exporter/${model_dir} \\\n",
    "  --json-instances=./test_sequences.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
